Add CI framework for Windows 64 (#229)

Though official support for Windows (64-bit) is blocked, we can
still add CI support for it. Once the Windows CI framework is
added to ACCP, we only have to reconfigure the
`tests/ci/run_windows_tests.bat` test script for ACCP windows
testing, then enable the CI support.

tests/ci/cdk/app.py

@@ -7,8 +7,9 @@
 
 from cdk.accp_github_ci_stack import ACCPGitHubCIStack
 from cdk.linux_docker_image_batch_build_stack import LinuxDockerImageBatchBuildStack
+from cdk.windows_docker_image_build_stack import WindowsDockerImageBuildStack
 from cdk.ecr_stack import EcrStack
-from util.metadata import AWS_ACCOUNT, AWS_REGION, LINUX_ECR_REPO
+from util.metadata import AWS_ACCOUNT, AWS_REGION, LINUX_ECR_REPO, WINDOWS_X86_ECR_REPO
 
 # Initialize app.
 app = core.App()
@@ -19,9 +20,13 @@
 # Define AWS ECR stacks.
 # ECR holds the docker images, which are pre-built to accelerate the code builds/tests of git pull requests.
 EcrStack(app, "accp-ecr-linux-all", LINUX_ECR_REPO, env=env)
+EcrStack(app, "accp-ecr-windows-x86", WINDOWS_X86_ECR_REPO, env=env)
 
 # Define CodeBuild Batch job for building Docker images.
 LinuxDockerImageBatchBuildStack(app, "accp-docker-image-build-linux", env=env)
+# AWS CodeBuild cannot build Windows Docker images because DIND (Docker In Docker) is not supported on Windows.
+# Windows Docker images are created by running commands in Windows EC2 instance.
+WindowsDockerImageBuildStack(app, "accp-docker-image-build-windows", env=env)
 
 # Define CodeBuild Batch job for testing code.
 x86_build_spec_file = "./cdk/codebuild/pr_integration_linux_x86_omnibus.yaml"
@@ -31,4 +36,10 @@
 extra_build_spec_file = "./cdk/codebuild/dieharder_overkill_omnibus.yaml"
 ACCPGitHubCIStack(app, "accp-ci-overkill-dieharder", LINUX_ECR_REPO, extra_build_spec_file, env=env)
 
+# TODO: Renable the code below when ACCP adds support for Windows.
+# Issue: https://github.com/corretto/amazon-corretto-crypto-provider/issues/48
+#
+# win_x86_build_spec_file = "./cdk/codebuild/pr_integration_windows_x86_omnibus.yaml"
+# ACCPGitHubCIStack(app, "accp-ci-pr-integration-windows-x86", WINDOWS_X86_ECR_REPO, win_x86_build_spec_file, env=env)
+
 app.synth()

tests/ci/cdk/cdk/codebuild/pr_integration_windows_x86_omnibus.yaml

@@ -0,0 +1,64 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+version: 0.2
+
+# Doc for batch https://docs.aws.amazon.com/codebuild/latest/userguide/batch-build-buildspec.html#build-spec.batch.build-list
+batch:
+  build-list:
+    - identifier: windows_msvc2015_corretto8_x64
+      buildspec: ./tests/ci/codebuild/windows-x86/windows-msvc2015.yml
+      env:
+        # https://docs.aws.amazon.com/codebuild/latest/userguide/build-env-ref-compute-types.html
+        type: WINDOWS_SERVER_2019_CONTAINER
+        privileged-mode: false
+        compute-type: BUILD_GENERAL1_LARGE
+        image: ECR_REPO_PLACEHOLDER:vs2015_corretto_latest
+        variables:
+          TEST_JAVA_HOME: C:\Program Files\Amazon Corretto\jdk1.8.0_332\bin
+    - identifier: windows_msvc2015_corretto11_x64
+      buildspec: ./tests/ci/codebuild/windows-x86/windows-msvc2015.yml
+      env:
+        type: WINDOWS_SERVER_2019_CONTAINER
+        privileged-mode: false
+        compute-type: BUILD_GENERAL1_LARGE
+        image: ECR_REPO_PLACEHOLDER:vs2015_corretto_latest
+        variables:
+          TEST_JAVA_HOME: C:\Program Files\Amazon Corretto\jdk11.0.15_9\bin
+    - identifier: windows_msvc2015_corretto17_x64
+      buildspec: ./tests/ci/codebuild/windows-x86/windows-msvc2015.yml
+      env:
+        type: WINDOWS_SERVER_2019_CONTAINER
+        privileged-mode: false
+        compute-type: BUILD_GENERAL1_LARGE
+        image: ECR_REPO_PLACEHOLDER:vs2015_corretto_latest
+        variables:
+          TEST_JAVA_HOME: C:\Program Files\Amazon Corretto\jdk17.0.3_6\bin
+
+    - identifier: windows_msvc2017_corretto8_x64
+      buildspec: ./tests/ci/codebuild/windows-x86/windows-msvc2017.yml
+      env:
+        type: WINDOWS_SERVER_2019_CONTAINER
+        privileged-mode: false
+        compute-type: BUILD_GENERAL1_LARGE
+        image: ECR_REPO_PLACEHOLDER:vs2017_corretto_latest
+        variables:
+          TEST_JAVA_HOME: C:\Program Files\Amazon Corretto\jdk1.8.0_332\bin
+    - identifier: windows_msvc2017_corretto11_x64
+      buildspec: ./tests/ci/codebuild/windows-x86/windows-msvc2017.yml
+      env:
+        type: WINDOWS_SERVER_2019_CONTAINER
+        privileged-mode: false
+        compute-type: BUILD_GENERAL1_LARGE
+        image: ECR_REPO_PLACEHOLDER:vs2017_corretto_latest
+        variables:
+          TEST_JAVA_HOME: C:\Program Files\Amazon Corretto\jdk11.0.15_9\bin
+    - identifier: windows_msvc2017_corretto17_x64
+      buildspec: ./tests/ci/codebuild/windows-x86/windows-msvc2017.yml
+      env:
+        type: WINDOWS_SERVER_2019_CONTAINER
+        privileged-mode: false
+        compute-type: BUILD_GENERAL1_LARGE
+        image: ECR_REPO_PLACEHOLDER:vs2017_corretto_latest
+        variables:
+          TEST_JAVA_HOME: C:\Program Files\Amazon Corretto\jdk17.0.3_6\bin

tests/ci/cdk/cdk/ssm/windows_docker_build_ssm_document.yaml

@@ -0,0 +1,27 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+---
+schemaVersion: '2.2'
+description: accp:buildWindowsDockerImages
+mainSteps:
+- action: aws:runPowerShellScript
+  name: runPowerShellScript
+  inputs:
+    timeoutSeconds: '7200'
+    runCommand:
+    - mkdir docker-images
+    - cd docker-images
+    # Install choco and git
+    - Set-ExecutionPolicy Bypass -Scope Process -Force; [Net.ServicePointManager]::SecurityProtocol = [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::Tls12; $env:chocolateyUseWindowsCompression = 'true'; [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072; iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1')) | Out-Null
+    - choco install git --version 2.23.0 -y
+    - $env:path+='C:\Program Files\Git\cmd'
+    # Git clone accp repo.
+    - git clone https://github.com/GITHUB_OWNER_PLACEHOLDER/amazon-corretto-crypto-provider.git
+    # Build Windows docker images.
+    - cd .\amazon-corretto-crypto-provider
+    - git checkout GITHUB_SOURCE_VERSION_PLACEHOLDER
+    - cd .\tests\ci\docker_images\windows
+    - Invoke-Expression -Command (Get-ECRLoginCommand -Region REGION_PLACEHOLDER).Command
+    - .\build_images.ps1
+    - .\push_images.ps1 ECR_PLACEHOLDER

tests/ci/cdk/cdk/windows_docker_image_build_stack.py

@@ -0,0 +1,62 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+from aws_cdk import core, aws_ec2 as ec2, aws_s3 as s3, aws_iam as iam, aws_ssm as ssm
+from util.iam_policies import ecr_power_user_policy_in_json, s3_read_write_policy_in_json
+from util.metadata import AWS_ACCOUNT, AWS_REGION, WINDOWS_X86_ECR_REPO, S3_BUCKET_NAME, GITHUB_REPO_OWNER, WIN_EC2_TAG_KEY, \
+    WIN_EC2_TAG_VALUE, SSM_DOCUMENT_NAME, GITHUB_SOURCE_VERSION
+from util.yml_loader import YmlLoader
+
+
+class WindowsDockerImageBuildStack(core.Stack):
+    """Define a temporary stack used to build Windows Docker images. After build, this stack will be destroyed."""
+
+    def __init__(self,
+                 scope: core.Construct,
+                 id: str,
+                 **kwargs) -> None:
+        super().__init__(scope, id, **kwargs)
+
+        # Define SSM command document.
+        ecr_repo = "{}.dkr.ecr.{}.amazonaws.com/{}".format(AWS_ACCOUNT, AWS_REGION, WINDOWS_X86_ECR_REPO)
+        placeholder_map = {"ECR_PLACEHOLDER": ecr_repo, "GITHUB_OWNER_PLACEHOLDER": GITHUB_REPO_OWNER,
+                           "REGION_PLACEHOLDER": AWS_REGION, "GITHUB_SOURCE_VERSION_PLACEHOLDER": GITHUB_SOURCE_VERSION}
+        content = YmlLoader.load("./cdk/ssm/windows_docker_build_ssm_document.yaml", placeholder_map)
+        ssm.CfnDocument(scope=self,
+                        id="{}-ssm-document".format(id),
+                        name=SSM_DOCUMENT_NAME,
+                        content=content,
+                        document_type="Command")
+
+        # Define a S3 bucket to store windows docker files and build scripts.
+        s3.Bucket(scope=self,
+                  id="{}-s3".format(id),
+                  bucket_name=S3_BUCKET_NAME,
+                  block_public_access=s3.BlockPublicAccess.BLOCK_ALL)
+
+        # Define a role for EC2.
+        ecr_power_user_policy = iam.PolicyDocument.from_json(ecr_power_user_policy_in_json([WINDOWS_X86_ECR_REPO]))
+        s3_read_write_policy = iam.PolicyDocument.from_json(s3_read_write_policy_in_json(S3_BUCKET_NAME))
+        inline_policies = {"ecr_power_user_policy": ecr_power_user_policy, "s3_read_write_policy": s3_read_write_policy}
+        role = iam.Role(scope=self, id="{}-role".format(id),
+                        assumed_by=iam.ServicePrincipal("ec2.amazonaws.com"),
+                        inline_policies=inline_policies,
+                        managed_policies=[
+                            iam.ManagedPolicy.from_aws_managed_policy_name("AmazonSSMManagedInstanceCore")
+                        ])
+
+        # Define Windows EC2 instance, where the SSM document will be executed.
+        machine_image = ec2.MachineImage.latest_windows(ec2.WindowsVersion.WINDOWS_SERVER_2019_ENGLISH_FULL_CONTAINERSLATEST)
+        vpc = ec2.Vpc(scope=self, id="{}-vpc".format(id))
+        block_device_volume = ec2.BlockDeviceVolume.ebs(volume_size=200, delete_on_termination=True)
+        block_device = ec2.BlockDevice(device_name="/dev/sda1", volume=block_device_volume)
+        instance = ec2.Instance(scope=self,
+                                id="{}-instance".format(id),
+                                instance_type=ec2.InstanceType(instance_type_identifier="m5d.xlarge"),
+                                vpc=vpc,
+                                role=role,
+                                block_devices=[block_device],
+                                vpc_subnets=ec2.SubnetSelection(subnet_type=ec2.SubnetType.PUBLIC),
+                                machine_image=machine_image)
+
+        core.Tags.of(instance).add(WIN_EC2_TAG_KEY, WIN_EC2_TAG_VALUE)

tests/ci/cdk/run-cdk.sh

@@ -98,8 +98,97 @@ function build_linux_docker_images() {
   linux_docker_img_build_status_check
 }
 
+
+function create_win_docker_img_build_stack() {
+  # Clean up build stacks if exists.
+  destroy_docker_img_build_stack
+  # Deploy accp ci stacks.
+  # When repeatedly deploy, error 'EIP failed Reason: Maximum number of addresses has been reached' can happen.
+  # https://forums.aws.amazon.com/thread.jspa?messageID=952368
+  # Workaround: go to AWS EIP console, release unused IP.
+  cdk deploy accp-docker-image-build-windows --require-approval never
+}
+
+function run_windows_img_build() {
+  # EC2 takes several minutes to be ready for running command.
+  echo "Wait 3 min for EC2 ready for SSM command execution."
+  sleep 180
+
+  # Run commands on windows EC2 instance to build windows docker images.
+  for i in {1..60}; do
+    instance_id=$(aws ec2 describe-instances \
+      --filters "Name=tag:${WIN_EC2_TAG_KEY},Values=${WIN_EC2_TAG_VALUE}" | jq -r '.Reservations[0].Instances[0].InstanceId')
+    if [[ "${instance_id}" == "null" ]]; then
+      sleep 60
+      continue
+    fi
+    instance_ping_status=$(aws ssm describe-instance-information \
+      --filters "Key=InstanceIds,Values=${instance_id}" | jq -r '.InstanceInformationList[0].PingStatus')
+    if [[ "${instance_ping_status}" == "Online" ]]; then
+      # https://awscli.amazonaws.com/v2/documentation/api/latest/reference/ssm/send-command.html
+      command_id=$(aws ssm send-command \
+        --instance-ids "${instance_id}" \
+        --document-name "${WIN_DOCKER_BUILD_SSM_DOCUMENT}" \
+        --output-s3-bucket-name "${S3_FOR_WIN_DOCKER_IMG_BUILD}" \
+        --output-s3-key-prefix 'runcommand' | jq -r '.Command.CommandId')
+      # Export for checking command run status.
+      export WINDOWS_DOCKER_IMG_BUILD_COMMAND_ID="${command_id}"
+      echo "Windows ec2 is executing SSM command."
+      return
+    else
+      echo "${i}: Current instance ping status: ${instance_ping_status}. Wait 1 minute to retry SSM command execution."
+      sleep 60
+    fi
+  done
+  echo "After 30 minutes, Windows ec2 is still not ready for SSM commands execution. Exit."
+  exit 1
+}
+
+function win_docker_img_build_status_check() {
+  export IMG_BUILD_STATUS='Failed'
+  # Every 5 min, this function checks if the windows docker image build is finished successfully.
+  # Normally, docker img build can take up to 1 hour. Here, we wait up to 30 * 5 min.
+  for i in {1..30}; do
+    # https://awscli.amazonaws.com/v2/documentation/api/latest/reference/ssm/list-commands.html
+    command_run_status=$(aws ssm list-commands --command-id "${WINDOWS_DOCKER_IMG_BUILD_COMMAND_ID}" | jq -r '.Commands[0].Status')
+    if [[ ${command_run_status} == 'Success' ]]; then
+      export IMG_BUILD_STATUS='Success'
+      echo "SSM command ${WINDOWS_DOCKER_IMG_BUILD_COMMAND_ID} finished successfully."
+      return
+    elif [[ ${command_run_status} == 'Failed' ]]; then
+      echo "SSM command ${WINDOWS_DOCKER_IMG_BUILD_COMMAND_ID} failed."
+      exit 1
+    else
+      echo "${i}: Wait 5 min for windows docker image build job finish."
+      sleep 300
+    fi
+  done
+  echo "SSM command ${WINDOWS_DOCKER_IMG_BUILD_COMMAND_ID} takes more time than expected."
+  exit 1
+}
+
+function build_win_docker_images() {
+  # Always destroy docker build stacks (which include EC2 instance) on EXIT.
+  trap destroy_docker_img_build_stack EXIT
+
+  # Create/update aws-ecr repo.
+  cdk deploy accp-ecr-windows-* --require-approval never
+
+  # Create aws windows build stack
+  create_win_docker_img_build_stack
+
+  echo "Executing AWS SSM commands to build Windows docker images."
+  run_windows_img_build
+
+  echo "Waiting for docker images creation. Building the docker images need to take 1 hour."
+  # TODO(CryptoAlg-624): These image build may fail due to the Docker Hub pull limits made on 2020-11-01.
+  win_docker_img_build_status_check
+}
+
+
 function setup_ci() {
   build_linux_docker_images
+  build_win_docker_images
 
   create_github_ci_stack
 }
@@ -150,6 +239,12 @@ function export_global_variables() {
   DATE_NOW="$(date +%Y-%m-%d-%H-%M)"
   export GITHUB_REPO='amazon-corretto-crypto-provider'
   export ECR_LINUX_REPO_NAME='accp-docker-images-linux'
+  export ECR_WINDOWS_X86_REPO_NAME='accp-docker-images-windows-x86'
+  export ACCP_S3_BUCKET_PREFIX='accp-windows-docker-image-build-s3'
+  export S3_FOR_WIN_DOCKER_IMG_BUILD="${ACCP_S3_BUCKET_PREFIX}-${DATE_NOW}"
+  export WIN_EC2_TAG_KEY='accp'
+  export WIN_EC2_TAG_VALUE="accp-windows-docker-image-build-${DATE_NOW}"
+  export WIN_DOCKER_BUILD_SSM_DOCUMENT="windows-ssm-document-${DATE_NOW}"
   export IMG_BUILD_STATUS='unknown'
 }
 
@@ -216,6 +311,9 @@ function main() {
   build-linux-img)
     build_linux_docker_images
     ;;
+  build-win-img)
+    build_win_docker_images
+    ;;
   synth)
     cdk synth accp-ci-*
     ;;

tests/ci/cdk/util/iam_policies.py

@@ -82,3 +82,24 @@ def ecr_power_user_policy_in_json(ecr_repo_names):
             }
         ]
     }
+
+def s3_read_write_policy_in_json(s3_bucket_name):
+    """
+    Define an IAM policy statement for reading and writing to S3 bucket.
+    :return: an IAM policy statement in json.
+    """
+    return {
+        "Version": "2012-10-17",
+        "Statement": [
+            {
+                "Effect": "Allow",
+                "Action": [
+                    "s3:Put*",
+                    "s3:Get*"
+                ],
+                "Resource": [
+                    "arn:aws:s3:::{}/*".format(s3_bucket_name)
+                ]
+            }
+        ]
+    }

tests/ci/cdk/util/metadata.py

@@ -11,8 +11,15 @@
 
 # Used when AWS CDK defines ECR repos.
 LINUX_ECR_REPO = EnvUtil.get("ECR_LINUX_REPO_NAME", "accp-docker-images-linux")
+WINDOWS_X86_ECR_REPO = EnvUtil.get("ECR_WINDOWS_X86_REPO_NAME", "accp-docker-images-windows-x86")
 
 # Used when AWS CodeBuild needs to create web_hooks.
 GITHUB_REPO_OWNER = EnvUtil.get("GITHUB_REPO_OWNER", "corretto")
 GITHUB_REPO_NAME = EnvUtil.get("GITHUB_REPO_NAME", "amazon-corretto-crypto-provider")
 GITHUB_SOURCE_VERSION = EnvUtil.get("GITHUB_SOURCE_VERSION", "develop")
+
+# Used when AWS CDK defines resources for Windows docker image build.
+S3_BUCKET_NAME = EnvUtil.get("S3_FOR_WIN_DOCKER_IMG_BUILD", "accp-windows-docker-image-build")
+WIN_EC2_TAG_KEY = EnvUtil.get("WIN_EC2_TAG_KEY", "accp")
+WIN_EC2_TAG_VALUE = EnvUtil.get("WIN_EC2_TAG_VALUE", "accp-windows-docker-image-build")
+SSM_DOCUMENT_NAME = EnvUtil.get("WIN_DOCKER_BUILD_SSM_DOCUMENT", "windows-ssm-document")

tests/ci/codebuild/windows-x86/windows-msvc2015.yml

@@ -0,0 +1,11 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0.
+
+version: 0.2
+
+phases:
+  build:
+    commands:
+      # vcvarsall will set the required lib and libpath for MSVC to compile everything. This is used for AWS-LC's
+      # Windows tests, but ACCP might use something different.
+      - .\tests\ci\run_windows_tests.bat "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat"

tests/ci/codebuild/windows-x86/windows-msvc2017.yml

@@ -0,0 +1,11 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0.
+
+version: 0.2
+
+phases:
+  build:
+    commands:
+      # vcvarsall will set the required lib and libpath for MSVC to compile everything. This is used for AWS-LC's
+      # Windows tests, but ACCP might use something different.
+      - .\tests\ci\run_windows_tests.bat "C:\Program Files (x86)\Microsoft Visual Studio\2017\BuildTools\VC\Auxiliary\Build\vcvarsall.bat"