Fix IMDSv2 token handling and docker permissions

README.md

@@ -85,10 +85,9 @@ The following table provides a sample cost breakdown for deploying this Guidance
 * An [AWS Identity and Access Management](http://aws.amazon.com/iam) (IAM) user with administrator access
 * [Configured AWS credentials](https://docs.aws.amazon.com/cdk/latest/guide/getting_started.html#getting_started_prerequisites)
 * [Docker](https://docs.docker.com/get-docker/), [Node.js](https://nodejs.org/en/download/)
-  , [Python 3.9](https://www.python.org/downloads/release/python-3916), [pip](https://pip.pypa.io/en/stable/installing/),
+  , [>=Python 3.11](https://www.python.org/downloads/release/python-3110/), [pip](https://pip.pypa.io/en/stable/installing/),
   and [jq](https://stedolan.github.io/jq/) installed on the workstation that you plan to deploy the guidance from.
 
-Note that the guidance is **only** compatible with Python 3.9.
 
 ### Deploy with AWS CDK
 
@@ -177,6 +176,27 @@ cdk deploy prodNitroSigner -O output.json
 
 Follow all subsequent steps from the dev deployment pointed out above.
 
+## Troubleshooting
+
+**Docker Image Push/Pull Error**
+* On `building` instance during `cdk deploy` step:
+```shell
+devNitroWalletEth: fail: docker push 012345678910.dkr.ecr.us-east-1.amazonaws.com/cdk-hnb659fds-container-assets-012345678910-us-east-1:ab3fe... exited with error code 1: failed commit on ref "manifest-sha256:7141...": unexpected status from PUT request to https://012345678910.dkr.ecr.us-east-1.amazonaws.com/v2/cdk-hnb659fds-container-assets-012345678910-us-east-1/manifests/ab3fe...: 400 Bad Request
+Failed to publish asset ab3fe...:012345678910-us-east-1
+```
+
+* On EC2 instance pulling docker container
+```shell
+ab3fe...: Pulling from cdk-hnb659fds-container-assets-012345678910-us-east-1
+unsupported media type application/vnd.in-toto+json
+```
+
+**Solution**
+* Issue might be related building and publishing docker containers from an `arm` based instances such as Apple Silicon, requiring docker `buildx` [issue](https://github.com/aws/aws-cdk/issues/30258)
+* Cleanup images from local docker repository (`docker rmi ...`) and from Amazon Elastic Container Registry (ECR) e.g. via AWS console
+* Set environment variable in terminal session (`export BUILDX_NO_DEFAULT_ATTESTATIONS=1`) or specify it during cdk deployment  (`BUILDX_NO_DEFAULT_ATTESTATIONS=1 cdk deploy`)
+
+
 ## Security
 
 See [CONTRIBUTING](CONTRIBUTING.md#security-issue-notifications) for more information.

application/eth2/lambda/layer/requirements.txt

@@ -1,3 +1,3 @@
-cryptography==39.0.2 ; python_version >= "3.9" and python_version < "4"
-requests>=2.28.1 ; python_version >= "3.9" and python_version < "4"
-urllib3<2; python_version >= "3.9" and python_version < "4"
+cryptography==41.0.7
+requests>=2.31.0
+urllib3==2.1.0

application/eth2/watchdog/watchdog.py

@@ -28,29 +28,51 @@
 _logger.addHandler(handler)
 
 
-def get_aws_session_token() -> dict:
+def get_imds_token():
     http_ec2_client = client.HTTPConnection("169.254.169.254")
-    http_ec2_client.request("GET", "/latest/meta-data/iam/security-credentials/")
-    r = http_ec2_client.getresponse()
+    headers = {
+        "X-aws-ec2-metadata-token-ttl-seconds": "21600"  # Token valid for 6 hours
+    }
+    http_ec2_client.request("PUT", "/latest/api/token", headers=headers)
+    token_response = http_ec2_client.getresponse()
+    return token_response.read().decode()
 
-    instance_profile_name = r.read().decode()
 
-    http_ec2_client = client.HTTPConnection("169.254.169.254")
-    http_ec2_client.request(
-        "GET",
-        f"/latest/meta-data/iam/security-credentials/{instance_profile_name}",
-    )
-    r = http_ec2_client.getresponse()
+def get_aws_session_token():
+    try:
+        token = get_imds_token()
 
-    response = json.loads(r.read())
+        http_ec2_client = client.HTTPConnection("169.254.169.254")
+        headers = {"X-aws-ec2-metadata-token": token}
 
-    credential = {
-        "access_key_id": response["AccessKeyId"],
-        "secret_access_key": response["SecretAccessKey"],
-        "token": response["Token"],
-    }
+        # Get instance profile name
+        http_ec2_client.request(
+            "GET",
+            "/latest/meta-data/iam/security-credentials/",
+            headers=headers
+        )
+        r = http_ec2_client.getresponse()
+        instance_profile_name = r.read().decode()
+
+        # Get credentials
+        http_ec2_client.request(
+            "GET",
+            f"/latest/meta-data/iam/security-credentials/{instance_profile_name}",
+            headers=headers
+        )
+        r = http_ec2_client.getresponse()
+        response = json.loads(r.read())
+        return {
+            "access_key_id": response["AccessKeyId"],
+            "secret_access_key": response["SecretAccessKey"],
+            "token": response["Token"],
+        }
 
-    return credential
+    except Exception as e:
+        raise Exception(f"Failed to retrieve instance credentials: {str(e)}")
+    finally:
+        if 'http_ec2_client' in locals():
+            http_ec2_client.close()
 
 
 def get_cloudformation_stack_id(cf_stack_name: str) -> str:
@@ -92,8 +114,8 @@ def nitro_cli_describe_call(name: str = None) -> bool:
             return False
 
         if (
-            response[0].get("EnclaveName") != name
-            and response[0].get("State") != "Running"
+                response[0].get("EnclaveName") != name
+                and response[0].get("State") != "Running"
         ):
             return False
 
@@ -230,19 +252,19 @@ def get_encrypted_tls_key(tls_keys_table_name: str, key_id=1) -> str:
 
 
 def init_web3signer_call(
-    tls_keys_table_name: str, cf_stack_name: str, validator_keys_table_name: str
+        tls_keys_table_name: str, cf_stack_name: str, validator_keys_table_name: str
 ) -> None:
     uuid = get_cloudformation_stack_id(cf_stack_name)
     encrypted_validator_keys = get_encrypted_validator_keys(
         validator_keys_table_name, uuid
     )
     encrypted_tls_key = get_encrypted_tls_key(tls_keys_table_name=tls_keys_table_name)
 
-    credential = get_aws_session_token()
+    credentials = get_aws_session_token()
 
     payload = {
         "operation": "init",
-        "credential": credential,
+        "credential": credentials,
         "encrypted_tls_key": encrypted_tls_key,
         "encrypted_validator_keys": encrypted_validator_keys,
     }

nitro_wallet/nitro_wallet_stack.py

@@ -94,13 +94,18 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None:
             "EthereumSigningServerImage",
             directory="./application/{}/server".format(application_type),
             build_args={"REGION_ARG": self.region, "LOG_LEVEL_ARG": log_level},
+            platform=ecr_assets.Platform.LINUX_AMD64,
+            asset_name="EthereumSigningServerImage"
+
         )
 
         signing_enclave_image = ecr_assets.DockerImageAsset(
             self,
             "EthereumSigningEnclaveImage",
             directory="./application/{}/enclave".format(application_type),
             build_args={"REGION_ARG": self.region, "LOG_LEVEL_ARG": log_level},
+            platform=ecr_assets.Platform.LINUX_AMD64,
+            asset_name="EthereumSigningEnclaveImage"
         )
 
         watchdog = s3_assets.Asset(
@@ -227,6 +232,7 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None:
             block_devices=[block_device],
             role=role,
             security_group=nitro_instance_sg,
+            http_put_response_hop_limit=3
         )
 
         nitro_nlb = elbv2.NetworkLoadBalancer(
@@ -292,7 +298,7 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None:
             self,
             "NitroInvokeLambdaLayer",
             entry="application/{}/lambda/layer".format(params["application_type"]),
-            compatible_runtimes=[lambda_.Runtime.PYTHON_3_9],
+            compatible_runtimes=[lambda_.Runtime.PYTHON_3_11],
         )
 
         invoke_lambda = lambda_python.PythonFunction(
@@ -301,7 +307,7 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None:
             entry="application/{}/lambda/NitroInvoke".format(params["application_type"]),
             handler="lambda_handler",
             index="lambda_function.py",
-            runtime=lambda_.Runtime.PYTHON_3_9,
+            runtime=lambda_.Runtime.PYTHON_3_11,
             timeout=Duration.minutes(2),
             memory_size=256,
             environment={
@@ -314,6 +320,7 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None:
             vpc=vpc,
             vpc_subnets=ec2.SubnetSelection(subnet_type=ec2.SubnetType.PRIVATE_WITH_EGRESS),
             security_groups=[signer_client_sg],
+            architecture=lambda_.Architecture.X86_64
         )
 
         encryption_key.grant_encrypt(invoke_lambda)

requirements-dev.txt

@@ -1,5 +1,5 @@
-pytest==6.2.5 ; python_version >= "3.9" and python_version < "3.10"
-black>=22.10.0 ; python_version >= "3.9" and python_version < "3.10"
-pre-commit>=2.20.0 ; python_version >= "3.9" and python_version < "3.10"
-bandit>=1.7.4 ; python_version >= "3.9" and python_version < "3.10"
-flake8==7.0.0 ; python_version >= "3.9" and python_version < "3.10"
+pytest==7.4.4
+black>=23.12.1
+pre-commit>=3.6.0
+bandit>=1.7.6
+flake8==7.0.0

requirements.txt

@@ -1,4 +1,4 @@
-aws-cdk-lib==2.51.1 ; python_version >= "3.9" and python_version < "3.10"
-constructs>=10.0.0,<11.0.0 ; python_version >= "3.9" and python_version < "3.10"
-aws-cdk.aws-lambda-python-alpha==2.51.0a0 ; python_version >= "3.9" and python_version < "3.10"
-cdk-nag>=2.21.11 ; python_version >= "3.9" and python_version < "3.10"
+aws-cdk-lib==2.98.0
+constructs==10.1.271
+aws-cdk.aws-lambda-python-alpha==2.51.0a0
+cdk-nag==2.27.88

scripts/load_validator_keys/load_validator_keys.py

@@ -37,20 +37,25 @@
 logger.addHandler(handler)
 logger.propagate = False
 
+region = os.getenv("CDK_DEPLOY_REGION", "us-east-1")
+
 kms_key_arn = os.getenv("KMS_KEY_ARN")
 table_name = os.getenv("DDB_TABLE_NAME")
 cf_stack_name = os.getenv("CF_STACK_NAME")
 
-client_kms = boto3.client("kms")
-dynamodb = boto3.resource("dynamodb")
+client_kms = boto3.client(service_name="kms",
+                          region_name=region)
+dynamodb = boto3.resource(service_name="dynamodb",
+                          region_name=region)
 
 words_list_path = "word_lists"
 
 
 def get_cloudformation_stack_id(cf_stack_name):
     """Get CF Stack ID"""
 
-    client = boto3.client(service_name="cloudformation")
+    client = boto3.client(service_name="cloudformation",
+                          region_name=region)
 
     try:
         response = client.describe_stacks(
@@ -78,12 +83,11 @@ def verify_keystore(credential: Credential, keystore: Keystore, password: str) -
 
 
 def main(
-    num_validators=5,
-    mnemonic_language="english",
-    chain="goerli",
-    eth1_withdrawal_address="0x6F4b46423fc6181a0cF34e6716c220BD4d6C2471",
+        num_validators=5,
+        mnemonic_language="english",
+        chain="sepolia",
+        eth1_withdrawal_address="0x6F4b46423fc6181a0cF34e6716c220BD4d6C2471",
 ) -> list:
-
     if kms_key_arn is None:
         raise ValueError("Specify KMS_KEY_ARN environment variable")
 

scripts/load_validator_keys/requirements.txt

@@ -1,22 +1,22 @@
-boto3>=1.26.14 ; python_version >= "3.9" and python_version < "3.10"
-botocore>=1.29.14 ; python_version >= "3.9" and python_version < "3.10"
-cached-property==1.5.2 ; python_version >= "3.9" and python_version < "3.10"
-click==8.1.3 ; python_version >= "3.9" and python_version < "3.10"
-cytoolz==0.11.2 ; python_version >= "3.9" and python_version < "3.10"
-eth-hash==0.3.2 ; python_version >= "3.9" and python_version < "3.10"
-eth-typing==2.3.0 ; python_version >= "3.9" and python_version < "3.10"
-eth-utils==1.10.0 ; python_version >= "3.9" and python_version < "3.10"
-jmespath==1.0.1 ; python_version >= "3.9" and python_version < "3.10"
-lru-dict==1.1.7 ; python_version >= "3.9" and python_version < "3.10"
-mypy-extensions==0.4.3 ; python_version >= "3.9" and python_version < "3.10"
-py-ecc==5.2.0 ; python_version >= "3.9" and python_version < "3.10"
-pycryptodome==3.14.1 ; python_version >= "3.9" and python_version < "3.10"
-pyrsistent==0.16.1 ; python_version >= "3.9" and python_version < "3.10"
-python-dateutil==2.8.2 ; python_version >= "3.9" and python_version < "3.10"
-s3transfer==0.6.0 ; python_version >= "3.9" and python_version < "3.10"
-six==1.16.0 ; python_version >= "3.9" and python_version < "3.10"
-ssz==0.2.4 ; python_version >= "3.9" and python_version < "3.10"
-staking-deposit @ git+https://github.com/ethereum/staking-deposit-cli.git@v2.3.0 ; python_version >= "3.9" and python_version < "3.10"
-toolz==0.11.2 ; python_version >= "3.9" and python_version < "3.10"
-urllib3==1.26.12 ; python_version >= "3.9" and python_version < "3.10"
-cryptography==39.0.2 ; python_version >= "3.9" and python_version < "3.10"
+boto3==1.34.14
+botocore==1.34.14
+cached-property==1.5.2
+click==8.1.7
+cytoolz==0.12.2
+eth-hash==0.5.2
+eth-typing==3.5.1
+eth-utils==2.3.1
+jmespath==1.0.1
+lru-dict==1.2.0
+mypy-extensions==1.0.0
+py-ecc==6.0.0
+pycryptodome==3.19.0
+pyrsistent>=0.16.0
+python-dateutil==2.8.2
+s3transfer==0.10.0
+six==1.16.0
+ssz==0.3.1
+staking-deposit @ git+https://github.com/ethereum/staking-deposit-cli.git@v2.8.0
+toolz==0.12.0
+urllib3<2.1.0
+cryptography==41.0.7

scripts/start_signing_service.sh

@@ -13,18 +13,36 @@ web3signer_init_flag_param_name=$(jq -r '."'${stack_name}'"."Web3SignerInitFlagP
 
 instance_ids=$(./scripts/get_asg_instances.sh ${asg_name} | tr "\n" " ")
 
-start_command_id=$(aws ssm send-command --region "${CDK_DEPLOY_REGION}" --document-name "AWS-RunShellScript" --instance-ids ${instance_ids} --parameters 'commands=["sudo systemctl start nitro-signing-server.service"]' | jq -r '.Command.CommandId')
+start_command_id=$(aws ssm send-command \
+ --region "${CDK_DEPLOY_REGION}" \
+ --document-name "AWS-RunShellScript" \
+ --instance-ids ${instance_ids} \
+ --parameters 'commands=["sudo systemctl start nitro-signing-server.service"]' | jq -r '.Command.CommandId')
 
 sleep 15
-status_command_id_hot=$(aws ssm send-command --region "${CDK_DEPLOY_REGION}" --document-name "AWS-RunShellScript" --instance-ids ${instance_ids} --parameters 'commands=["sudo systemctl status nitro-signing-server.service"]' | jq -r '.Command.CommandId')
+status_command_id_hot=$(aws ssm send-command \
+ --region "${CDK_DEPLOY_REGION}" \
+ --document-name "AWS-RunShellScript" \
+ --instance-ids ${instance_ids} \
+ --parameters 'commands=["sudo systemctl status nitro-signing-server.service"]' | jq -r '.Command.CommandId')
 
 instance_ids_nl=$(echo ${instance_ids} | tr "\n " " ")
 for instance_id in ${instance_ids_nl}; do
-  status=$(aws ssm list-command-invocations --instance-id ${instance_id} --command-id ${status_command_id_hot} --details | jq -r '.CommandInvocations[0].CommandPlugins[0].Output')
+  status=$(aws ssm list-command-invocations \
+  --region ${CDK_DEPLOY_REGION} \
+  --instance-id ${instance_id} \
+  --command-id ${status_command_id_hot} \
+  --details | jq -r '.CommandInvocations[0].CommandPlugins[0].Output')
   echo "${instance_id}:"
   echo ${status}
 done
 
-aws ssm put-parameter --name "${web3signer_init_flag_param_name}" --type "String" --value "true" --overwrite
+aws ssm put-parameter \
+ --name "${web3signer_init_flag_param_name}" \
+ --type "String" \
+ --value "true" \
+ --overwrite \
+ --region ${CDK_DEPLOY_REGION} \
+ --no-cli-pager
 
 printf "\n%s\n" "($(date '+%d/%m/%Y %H:%M:%S')) service has been started and is healthy"