diff --git a/terraform/.terraform.lock.hcl b/terraform/.terraform.lock.hcl index 3370026..6aa9f4d 100644 --- a/terraform/.terraform.lock.hcl +++ b/terraform/.terraform.lock.hcl @@ -2,19 +2,8 @@ # Manual edits may be lost in future updates. provider "registry.terraform.io/hashicorp/aws" { - version = "3.74.0" + version = "4.54.0" hashes = [ - "h1:y9b9LluBQGrZHURGY1Xmrhb+zQ6qRit3oqFSLijkGe0=", - "zh:00767509c13c0d1c7ad6af702c6942e6572aa6d529b40a00baacc0e73faafea2", - "zh:03aafdc903ad49c2eda03889f927f44212674c50e475a9c6298850381319eec2", - "zh:2de8a6a97b180f909d652f215125aa4683e99db15fcf3b28d62e3d542f875ed6", - "zh:3ac29ebc3af99028f4230a79f56606a0c2954b68767bd749b921a76eb4f3bd30", - "zh:50add2e2d118a15a644360eabc5a34cec59f2560b491f8fabf9c52ab83ca7b09", - "zh:85dd8e81910ab79f841a4a595fdd8ac358fbfe460956144afb0be3d81f91fe10", - "zh:895de83d0f0941fde31bfc53fa6b1ea276901f006bec221bbdee4771a04f3693", - "zh:a15c9724aac52d1ba5001d2d83e42843099b52b1638ea29d84e20be0f45fa4f1", - "zh:c982a64463bd73e9bff2589de214b1de0a571438d9015001f9eae45cfc3a2559", - "zh:e9ef973c18078324e43213ea1252c12b9441e566bf054ddfdbff5dd62f3035d9", - "zh:f297e705b0f339c8baa27ae70db5df9aa6578adfe1ea3d2ba8edc186512464eb", + "h1:CvZ2TNI0ImPJN1xxCCZsrU5mEQXcyfPcEWM7u7tMfPs=", ] } diff --git a/terraform/buildkite-v5.10.0.yml b/terraform/buildkite-v5.10.0.yml deleted file mode 100644 index 9266833..0000000 --- a/terraform/buildkite-v5.10.0.yml +++ /dev/null @@ -1,1233 +0,0 @@ ---- -AWSTemplateFormatVersion: "2010-09-09" -Description: "Buildkite stack v5.10.0" - -# The Buildkite Elastic CI Stack for AWS gives you a private, -# autoscaling Buildkite Agent cluster. Use it to parallelize -# large test suites across thousands of nodes, run tests and -# deployments for Linux or Windows based services and apps, -# or run AWS ops tasks. -# -# To gain a better understanding of how Elastic CI Stack works -# and how to use it most effectively and securely, check out -# the following resources: -# -# * Elastic CI Stack for AWS Overview: https://buildkite.com/docs/agent/v3/elastic_ci_aws -# * Elastic CI Stack for AWS Tutorial: https://buildkite.com/docs/tutorials/elastic-ci-stack-aws -# * Running Buildkite Agent on AWS: https://buildkite.com/docs/agent/v3/aws -# * GitHub Repo for Elastic CI Stack: https://github.com/buildkite/elastic-ci-stack-for-aws -# * Template Parameters for Elastic CI Stack for AWS: https://buildkite.com/docs/agent/v3/elastic-ci-aws/parameters -# * Using AWS Secrets Manager: https://buildkite.com/docs/agent/v3/aws/secrets-manager -# * VPC Design: https://buildkite.com/docs/agent/v3/aws/vpc -# * CloudFormation Service Role: https://buildkite.com/docs/agent/v3/elastic-ci-aws/cloudformation-service-role - -Transform: AWS::Serverless-2016-10-31 - -Metadata: - AWS::CloudFormation::Interface: - ParameterGroups: - - Label: - default: Buildkite Configuration - Parameters: - - BuildkiteAgentTokenParameterStorePath - - BuildkiteAgentTokenParameterStoreKMSKey - - BuildkiteAgentToken - - BuildkiteQueue - - - Label: - default: Advanced Buildkite Configuration - Parameters: - - BuildkiteAgentRelease - - BuildkiteAgentTags - - BuildkiteAgentTimestampLines - - BuildkiteAgentExperiments - - BuildkiteTerminateInstanceAfterJob - - BuildkiteAdditionalSudoPermissions - - BuildkiteWindowsAdministrator - - - Label: - default: Network Configuration - Parameters: - - VpcId - - Subnets - - AvailabilityZones - - SecurityGroupId - - AssociatePublicIpAddress - - - Label: - default: Instance Configuration - Parameters: - - ImageId - - ImageIdParameter - - InstanceType - - EnableInstanceStorage - - AgentsPerInstance - - KeyName - - SpotPrice - - SecretsBucket - - SecretsBucketRegion - - ArtifactsBucket - - AuthorizedUsersUrl - - BootstrapScriptUrl - - RootVolumeSize - - RootVolumeName - - RootVolumeType - - ManagedPolicyARN - - InstanceRoleName - - IMDSv2Tokens - - EnableDetailedMonitoring - - - Label: - default: Auto-scaling Configuration - Parameters: - - MinSize - - MaxSize - - OnDemandPercentage - - ScaleOutFactor - - ScaleInIdlePeriod - - ScaleOutForWaitingJobs - - InstanceCreationTimeout - - - Label: - default: Cost Allocation Configuration - Parameters: - - EnableCostAllocationTags - - CostAllocationTagName - - CostAllocationTagValue - - - Label: - default: Docker Daemon Configuration - Parameters: - - EnableDockerUserNamespaceRemap - - EnableDockerExperimental - - - Label: - default: Docker Registry Configuration - Parameters: - - ECRAccessPolicy - - - Label: - default: Plugin Configuration - Parameters: - - EnableSecretsPlugin - - EnableECRPlugin - - EnableDockerLoginPlugin - -Parameters: - KeyName: - Description: Optional - SSH keypair used to access the buildkite instances via ec2_user, setting this will enable SSH ingress - Type: String - Default: "" - - BuildkiteAgentRelease: - Type: String - AllowedValues: - - stable - - beta - - edge - Default: "stable" - - BuildkiteAgentToken: - Description: Buildkite agent registration token. Or, preload it into SSM Parameter Store and use BuildkiteAgentTokenParameterStorePath for secure environments. - Type: String - NoEcho: true - Default: "" - - BuildkiteAgentTokenParameterStorePath: - Description: Existing SSM Parameter Store path to the Buildkite agent registration token (takes precedence over BuildkiteAgentToken). Expects a leading slash ('/'). - Type: String - Default: "" - AllowedPattern: "^$|^/[a-zA-Z0-9_.\\-/]+$" - ConstraintDescription: "Expects a leading forward slash" - - BuildkiteAgentTokenParameterStoreKMSKey: - Description: AWS KMS key ID used to encrypt the SSM parameter (if encrypted) - Type: String - Default: "" - - BuildkiteAgentTags: - Description: Additional tags separated by commas to provide to the agent. E.g os=linux,llamas=always - Type: String - Default: "" - - BuildkiteAgentTimestampLines: - Description: Set to true to prepend timestamps to every line of output - Type: String - AllowedValues: - - "true" - - "false" - Default: "false" - - BuildkiteAgentExperiments: - Description: Agent experiments to enable, comma delimited. See https://github.com/buildkite/agent/blob/master/EXPERIMENTS.md. - Type: String - Default: "" - - BuildkiteTerminateInstanceAfterJob: - Description: Set to "true" to terminate the instance after a job has completed. - Type: String - AllowedValues: - - "true" - - "false" - Default: "false" - - BuildkiteAdditionalSudoPermissions: - Description: Optional - Comma separated list of commands to allow the buildkite-agent user to run using sudo. - Type: String - Default: "" - - BuildkiteWindowsAdministrator: - Description: Set to "true" to add the local "buildkite-agent" user account to the local Windows Administrator group. - Type: String - AllowedValues: - - "true" - - "false" - Default: "true" - - BuildkiteQueue: - Description: Queue name that agents will use, targeted in pipeline steps using "queue={value}" - Type: String - Default: default - MinLength: 1 - - AgentsPerInstance: - Description: Number of Buildkite agents to run on each instance - Type: Number - Default: 1 - MinValue: 1 - - SecretsBucket: - Description: Optional - Name of an existing S3 bucket containing pipeline secrets (Created if left blank) - Type: String - Default: "" - - SecretsBucketRegion: - Description: Optional - Region for the SecretsBucket. If blank the bucket's region is dynamically discovered. - Type: String - Default: "" - - ArtifactsBucket: - Description: Optional - Name of an existing S3 bucket for build artifact storage - Type: String - Default: "" - - BootstrapScriptUrl: - Description: Optional - HTTPS or S3 URL to run on each instance during boot - Type: String - Default: "" - - AuthorizedUsersUrl: - Description: Optional - HTTPS or S3 URL to periodically download ssh authorized_keys from, setting this will enable SSH ingress. authorized_keys are applied to ec2_user - Type: String - Default: "" - - VpcId: - Type: String - Description: Optional - Id of an existing VPC to launch instances into. Leave blank to have a new VPC created - Default: "" - - Subnets: - Type: CommaDelimitedList - Description: Optional - Comma separated list of two existing VPC subnet ids where EC2 instances will run. Required if setting VpcId. - Default: "" - - AvailabilityZones: - Type: CommaDelimitedList - Description: Optional - Comma separated list of AZs that subnets are created in (if Subnets parameter is not specified) - Default: "" - - InstanceType: - Description: Instance type. Comma-separated list with 1-4 instance types. The order is a prioritized preference for launching OnDemand instances, and a non-prioritized list of types to consider for Spot Instances (where used). - Type: String - Default: t3.large - MinLength: 1 - AllowedPattern: "^[\\w\\.]+(,[\\w\\.]*){0,3}$" - ConstraintDescription: "must contain 1-4 instance types separated by commas. No space before/after the comma." - - SpotPrice: - Description: Maximum spot price to use for the instances, in instance cost per hour. Values >0 will result in 100% of instances being spot. 0 means only use normal (non-spot) instances. This parameter is deprecated - we recommend setting to 0 and using OnDemandPercentage to opt into spot instances. - Type: String - Default: 0 - - MaxSize: - Description: Maximum number of instances - Type: Number - Default: 10 - MinValue: 1 - - MinSize: - Description: Minimum number of instances - Type: Number - Default: 0 - - OnDemandPercentage: - Description: Percentage of total instances that should launch as OnDemand. Default is 100% OnDemand - reduce this to use some Spot Instances when they're available and cheaper than the OnDemand price. A value of 70 means 70% OnDemand and 30% Spot Instances. - Type: Number - Default: 100 - MinValue: 0 - MaxValue: 100 - - ScaleOutFactor: - Description: A decimal factor to apply to scale out changes to speed up or slow down scale-out - Type: Number - Default: 1.0 - - ScaleInIdlePeriod: - Description: Number of seconds an agent must be idle before terminating - Type: Number - Default: 600 - - ScaleOutForWaitingJobs: - Type: String - Description: Whether to scale-out for steps behind wait steps. Make sure you have a long enough idle period! - AllowedValues: - - "true" - - "false" - Default: "false" - - InstanceCreationTimeout: - Description: Timeout period for Autoscaling Group Creation Policy - Type: String - Default: "" - - RootVolumeSize: - Description: Size of each instance's root EBS volume (in GB) - Type: Number - Default: 250 - MinValue: 10 - - RootVolumeName: - Description: Name of the root block device for your AMI - Type: String - Default: "" - - RootVolumeType: - Description: Type of root volume to use - Type: String - Default: "gp3" - - SecurityGroupId: - Type: String - Description: Optional - Comma separated list of security group ids to assign to instances - Default: "" - - ImageId: - Type: String - Description: Optional - Custom AMI to use for instances (must be based on the stack's AMI) - Default: "" - - ImageIdParameter: - Type: String - Description: Optional - Custom AMI SSM Parameter to use for instances (must be based on the stack's AMI) - Default: "" - - ManagedPolicyARN: - Type: CommaDelimitedList - Description: Optional - Comma separated list of managed IAM policy ARNs to attach to the instance role - Default: "" - - IMDSv2Tokens: - Type: String - Description: Whether IMDSv2 tokens must be used for the Instance Metadata Service. - AllowedValues: - - optional - - required - Default: optional - - InstanceRoleName: - Type: String - Description: Optional - A name for the IAM Role attached to the Instance Profile - Default: "" - - InstanceRolePermissionsBoundaryARN: - Type: String - Description: The ARN of the policy used to set the permissions boundary for the role. - Default: "" - - InstanceOperatingSystem: - Type: String - Description: The operating system to run on the instances - AllowedValues: - - linux - - windows - Default: "linux" - - ECRAccessPolicy: - Type: String - Description: ECR access policy to give container instances - AllowedValues: - - none - - readonly - - poweruser - - full - Default: "none" - - AssociatePublicIpAddress: - Type: String - Description: Associate instances with public IP addresses - AllowedValues: - - "true" - - "false" - Default: "true" - - EnableSecretsPlugin: - Type: String - Description: Enables s3-secrets plugin for all pipelines - AllowedValues: - - "true" - - "false" - Default: "true" - - EnableECRPlugin: - Type: String - Description: Enables ecr plugin for all pipelines - AllowedValues: - - "true" - - "false" - Default: "true" - - EnableDockerLoginPlugin: - Type: String - Description: Enables docker-login plugin for all pipelines - AllowedValues: - - "true" - - "false" - Default: "true" - - EnableDockerUserNamespaceRemap: - Type: String - Description: Enables Docker user namespace remapping so docker runs as buildkite-agent - AllowedValues: - - "true" - - "false" - Default: "true" - - EnableDockerExperimental: - Type: String - Description: Enables Docker experimental features - AllowedValues: - - "true" - - "false" - Default: "false" - - EnableInstanceStorage: - Type: String - Description: Mount available NVMe Instance Storage at /mnt/ephemeral - AllowedValues: - - "true" - - "false" - Default: "false" - - EnableCostAllocationTags: - Type: String - Description: Enables AWS Cost Allocation tags for all resources in the stack. See https://docs.aws.amazon.com/awsaccountbilling/latest/aboutv2/cost-alloc-tags.html - AllowedValues: - - "true" - - "false" - Default: "false" - - CostAllocationTagName: - Type: String - Description: The name of the Cost Allocation Tag used for billing purposes - Default: "CreatedBy" - - CostAllocationTagValue: - Type: String - Description: The value of the Cost Allocation Tag used for billing purposes - Default: "buildkite-elastic-ci-stack-for-aws" - - EnableAgentGitMirrorsExperiment: - Type: String - Description: Enables the git-mirrors experiment in the agent - AllowedValues: - - "true" - - "false" - Default: "false" - - EnableDetailedMonitoring: - Type: String - Description: Enable detailed EC2 monitoring - AllowedValues: - - "true" - - "false" - Default: "false" - -Rules: - HasToken: - Assertions: - - Assert: - !Or - - !Not - - !Equals - - !Ref BuildkiteAgentToken - - "" - - !Not - - !Equals - - !Ref BuildkiteAgentTokenParameterStorePath - - "" - AssertDescription: "You must provide BuildkiteAgentToken or BuildkiteAgentTokenParameterStorePath" - -Outputs: - VpcId: - Value: - !If [ CreateVpcResources, !Ref Vpc, !Ref VpcId ] - Export: - Name: !Sub '${AWS::StackName}-VpcId' - - ManagedSecretsBucket: - Value: - !If [ CreateSecretsBucket, !Ref ManagedSecretsBucket, "Undefined" ] - Export: - Name: !Sub '${AWS::StackName}-ManagedSecretsBucket' - - ManagedSecretsLoggingBucket: - Value: - !If [ CreateSecretsBucket, !Ref ManagedSecretsLoggingBucket, "Undefined" ] - Export: - Name: !Sub '${AWS::StackName}-ManagedSecretsLoggingBucket' - - AutoScalingGroupName: - Value: !Ref AgentAutoScaleGroup - Export: - Name: !Sub '${AWS::StackName}-AutoScalingGroupName' - - InstanceRoleName: - Value: !Ref IAMRole - Export: - Name: !Sub '${AWS::StackName}-InstanceRoleName' - -Conditions: - SpotPriceSet: - !Not [ !Equals [ !Ref SpotPrice, 0 ] ] - - CreateVpcResources: - !Equals [ !Ref VpcId, "" ] - - CreateSecurityGroup: - !Equals [ !Ref SecurityGroupId, "" ] - - CreateSecretsBucket: - !And - - !Equals [ !Ref EnableSecretsPlugin, "true"] - - !Equals [ !Ref SecretsBucket, "" ] - - SetInstanceRoleName: - !Not [ !Equals [ !Ref InstanceRoleName, "" ] ] - - SetInstanceRolePermissionsBoundaryARN: - !Not [ !Equals [ !Ref InstanceRolePermissionsBoundaryARN, "" ] ] - - UseSpecifiedSecretsBucket: - !Not [ !Equals [ !Ref SecretsBucket, "" ] ] - - HasSecretsBucket: - !Or [ !Condition CreateSecretsBucket, !Condition UseSpecifiedSecretsBucket ] - - UseSpecifiedAvailabilityZones: - !Not [ !Equals [ !Join [ "", !Ref AvailabilityZones ], "" ] ] - - UseArtifactsBucket: - !Not [ !Equals [ !Ref ArtifactsBucket, "" ] ] - - HasImageId: - !Not [ !Equals [ !Ref ImageId, "" ] ] - HasImageIdParameter: - !Not [ !Equals [ !Ref ImageIdParameter, "" ] ] - - UseDefaultInstanceCreationTimeout: - !Equals [ !Ref InstanceCreationTimeout, "" ] - - UseDefaultRootVolumeName: - !Equals [ !Ref RootVolumeName, "" ] - - UseInstanceType2: - !Not [ !Equals [ !Select [ "1", !Split [ ",", !Join [ ",", [ !Ref InstanceType, "", "", "" ] ] ] ], ""] ] - - UseInstanceType3: - !Not [ !Equals [ !Select [ "2", !Split [ ",", !Join [ ",", [ !Ref InstanceType, "", "", "" ] ] ] ], ""] ] - - UseInstanceType4: - !Not [ !Equals [ !Select [ "3", !Split [ ",", !Join [ ",", [ !Ref InstanceType, "", "", "" ] ] ] ], ""] ] - - UseManagedPolicyARN: - !Not [ !Equals [ !Join [ "", !Ref ManagedPolicyARN ], "" ] ] - - UseECR: - !Not [ !Equals [ !Ref ECRAccessPolicy, "none" ] ] - - UseCustomerManagedParameterPath: - !Not [ !Equals [ !Ref BuildkiteAgentTokenParameterStorePath, "" ] ] - UseCustomerManagedKeyForParameterStore: - !Not [ !Equals [ !Ref BuildkiteAgentTokenParameterStoreKMSKey, "" ] ] - CreateAgentTokenParameter: - !Equals [ !Ref BuildkiteAgentTokenParameterStorePath, "" ] - - HasVariableSize: - !Not [ !Equals [ !Ref MaxSize, !Ref MinSize ] ] - - UseCostAllocationTags: - !Equals [ !Ref EnableCostAllocationTags, "true" ] - - HasKeyName: - !Not [ !Equals [ !Ref KeyName, "" ] ] - - EnableSshIngress: - !And - - { Condition : CreateSecurityGroup } - # Enable ingress if a key can be specified another way - - !Or - - { Condition: HasKeyName } - - !Not [ !Equals [ !Ref AuthorizedUsersUrl, "" ] ] - - # Whether or not there's any managed polices to attach - HasManagedPolicies: - !Or [ { Condition: UseManagedPolicyARN }, { Condition: UseECR } ] - - UseWindowsAgents: - !Equals [ !Ref InstanceOperatingSystem, "windows" ] - - UseLinuxAgents: - !Equals [ !Ref InstanceOperatingSystem, "linux" ] - - # Unfortunately, Cloudformation's !Or intrinsic function only accepts - # between 2 and 10 arguments. To get around this, we're grouping the - # instance families in sub-conditionals. At least this doesn't force us - # into using a Custom Resource. - UsingArmInstances: - !Or - - !Equals [ !Select [ 0, !Split [ ".", !Ref InstanceType ] ], "a1" ] - - !Or - - !Equals [ !Select [ 0, !Split [ ".", !Ref InstanceType ] ], "c6g" ] - - !Equals [ !Select [ 0, !Split [ ".", !Ref InstanceType ] ], "c6gd" ] - - !Equals [ !Select [ 0, !Split [ ".", !Ref InstanceType ] ], "c6gn" ] - - !Equals [ !Select [ 0, !Split [ ".", !Ref InstanceType ] ], "c7g" ] - - !Equals [ !Select [ 0, !Split [ ".", !Ref InstanceType ] ], "g5g" ] - - !Or - - !Equals [ !Select [ 0, !Split [ ".", !Ref InstanceType ] ], "Im4gn" ] - - !Equals [ !Select [ 0, !Split [ ".", !Ref InstanceType ] ], "Is4gen" ] - - !Or - - !Equals [ !Select [ 0, !Split [ ".", !Ref InstanceType ] ], "m6g" ] - - !Equals [ !Select [ 0, !Split [ ".", !Ref InstanceType ] ], "m6gd" ] - - !Or - - !Equals [ !Select [ 0, !Split [ ".", !Ref InstanceType ] ], "r6g" ] - - !Equals [ !Select [ 0, !Split [ ".", !Ref InstanceType ] ], "r6gd" ] - - !Equals [ !Select [ 0, !Split [ ".", !Ref InstanceType ] ], "t4g" ] - - !Equals [ !Select [ 0, !Split [ ".", !Ref InstanceType ] ], "x2gd" ] - -Mappings: - ECRManagedPolicy: - none : { Policy: '' } - readonly : { Policy: 'arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly' } - poweruser : { Policy: 'arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryPowerUser' } - full : { Policy: 'arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryFullAccess' } - - # Generated from Makefile via build/mappings.yml - AWSRegion2AMI: - us-east-1 : { linuxamd64: ami-0f39c6ae1dd91e580, linuxarm64: ami-0aeab800372121d9b, windows: ami-02a617663a9b3fcf4 } - us-east-2 : { linuxamd64: ami-0a55c0a389972afb5, linuxarm64: ami-09a9700fae1a701c3, windows: ami-0af5a62debae81021 } - us-west-1 : { linuxamd64: ami-0f0acff5b9d9ae848, linuxarm64: ami-084bb3f40c034b6f9, windows: ami-09bc1b31e5721e8c1 } - us-west-2 : { linuxamd64: ami-015e6c540e066cbb5, linuxarm64: ami-07f9d9a13422faebd, windows: ami-0abb308b033b12c06 } - af-south-1 : { linuxamd64: ami-0892c0debfe550759, linuxarm64: ami-091999951317f62a8, windows: ami-0b654bcadfab23fe3 } - ap-east-1 : { linuxamd64: ami-00582f2ed23dc0d95, linuxarm64: ami-0f3dc620cedb34ca0, windows: ami-08a867a4b506c15fc } - ap-south-1 : { linuxamd64: ami-08517a6ed510e97b5, linuxarm64: ami-025b4f0882e51a8ae, windows: ami-00f7059be75cfffdc } - ap-northeast-2 : { linuxamd64: ami-001400b8c5ed86cd6, linuxarm64: ami-0fcc19808efad1ba2, windows: ami-058a1e0f3e30adc4e } - ap-northeast-1 : { linuxamd64: ami-0a15ab8114ebc5f41, linuxarm64: ami-00f642706bdaf0c64, windows: ami-0a217836446cd442b } - ap-southeast-2 : { linuxamd64: ami-02a0d7e43aab73a84, linuxarm64: ami-0508af6f9aa018ac6, windows: ami-0ae288773fe0491f5 } - ap-southeast-1 : { linuxamd64: ami-0b3b208c65e3a1be0, linuxarm64: ami-09012d224ec82620e, windows: ami-03c5292687115026e } - ca-central-1 : { linuxamd64: ami-09bb0a2322ad1e113, linuxarm64: ami-0bf9084f5454c26a2, windows: ami-02ae7379603b1ca60 } - eu-central-1 : { linuxamd64: ami-024f8e253aa35981f, linuxarm64: ami-0df05b043eaa1640a, windows: ami-0d2ff86f61a002cf8 } - eu-west-1 : { linuxamd64: ami-0ad412a7859d1fcca, linuxarm64: ami-0a1b2dd29af9018ed, windows: ami-08e2fcee943ded860 } - eu-west-2 : { linuxamd64: ami-07f0b24b79d47e05a, linuxarm64: ami-0a1920a514cf8af92, windows: ami-06b7fc27155636fd3 } - eu-south-1 : { linuxamd64: ami-07478687996b5a60f, linuxarm64: ami-0e9b52dbcc9f0f1a6, windows: ami-04654d77172636cca } - eu-west-3 : { linuxamd64: ami-098c5849017c629b1, linuxarm64: ami-041079f5aecaecbcc, windows: ami-07a2682aa1efb76af } - eu-north-1 : { linuxamd64: ami-06cf6c3df01d8cb7c, linuxarm64: ami-06f9e9fc89950bae9, windows: ami-01231381a3f224a29 } - me-south-1 : { linuxamd64: ami-0a5f65aacc2c10903, linuxarm64: ami-05c47c58dbdf7668f, windows: ami-0da3501ec610f2735 } - sa-east-1 : { linuxamd64: ami-0fbdb79e495b69ad5, linuxarm64: ami-0c611a0f350d6ef18, windows: ami-0302b86d82ab143fb } - -Resources: - Vpc: - Type: AWS::EC2::VPC - Condition: CreateVpcResources - Properties: - CidrBlock: 10.0.0.0/16 - InstanceTenancy: default - Tags: - - Key: Name - Value: !Ref 'AWS::StackName' - - Gateway: - Type: AWS::EC2::InternetGateway - Condition: CreateVpcResources - Properties: - Tags: - - Key: Name - Value: !Ref 'AWS::StackName' - - GatewayAttachment: - Type: AWS::EC2::VPCGatewayAttachment - Condition: CreateVpcResources - Properties: - InternetGatewayId: !Ref Gateway - VpcId: !Ref Vpc - - Subnet0: - Type: AWS::EC2::Subnet - Condition: CreateVpcResources - DependsOn: - - GatewayAttachment - Properties: - AvailabilityZone: - !If - - "UseSpecifiedAvailabilityZones" - - !Select [ 0, !Ref AvailabilityZones ] - - !Select [ 0, !GetAZs '' ] - CidrBlock: 10.0.1.0/24 - VpcId: !Ref Vpc - Tags: - - Key: Name - Value: !Ref 'AWS::StackName' - - Subnet1: - Type: AWS::EC2::Subnet - Condition: CreateVpcResources - DependsOn: - - GatewayAttachment - Properties: - AvailabilityZone: - !If - - "UseSpecifiedAvailabilityZones" - - !Select [ 1, !Ref AvailabilityZones ] - - !Select [ 1, !GetAZs '' ] - CidrBlock: 10.0.2.0/24 - VpcId: !Ref Vpc - Tags: - - Key: Name - Value: !Ref 'AWS::StackName' - - Routes: - Type: AWS::EC2::RouteTable - Condition: CreateVpcResources - Properties: - VpcId: !Ref Vpc - Tags: - - Key: Name - Value: !Ref 'AWS::StackName' - - RouteDefault: - Type: AWS::EC2::Route - Condition: CreateVpcResources - DependsOn: - - GatewayAttachment - Properties: - DestinationCidrBlock: 0.0.0.0/0 - GatewayId: !Ref Gateway - RouteTableId: !Ref Routes - - Subnet0Routes: - Type: AWS::EC2::SubnetRouteTableAssociation - Condition: CreateVpcResources - Properties: - SubnetId: !Ref Subnet0 - RouteTableId: !Ref Routes - - Subnet1Routes: - Type: AWS::EC2::SubnetRouteTableAssociation - Condition: CreateVpcResources - Properties: - SubnetId: !Ref Subnet1 - RouteTableId: !Ref Routes - - # A resource that depends on the leaf nodes of the VPC configuration so that - # a strict ordering can be established on teardown. - VpcComplete: - Type: AWS::CloudFormation::WaitConditionHandle - Metadata: - VpcResources: !If - - CreateVpcResources - - [ !Ref RouteDefault, !Ref Subnet0Routes, !Ref Subnet1Routes ] - - !Ref "AWS::NoValue" - - BuildkiteAgentTokenParameter: - Type: AWS::SSM::Parameter - Condition: CreateAgentTokenParameter - Properties: - Name: !Sub "/${AWS::StackName}/buildkite/agent-token" - Type: String - Value: !Ref BuildkiteAgentToken - - # Allow ec2 instances to assume a role and be granted the IAMPolicies - IAMInstanceProfile: - Type: AWS::IAM::InstanceProfile - Properties: - Path: / - Roles: [ !Ref IAMRole ] - - IAMRole: - Type: AWS::IAM::Role - Properties: - RoleName: !If [ SetInstanceRoleName, !Ref InstanceRoleName, !Sub "${AWS::StackName}-Role" ] - PermissionsBoundary: !If [ SetInstanceRolePermissionsBoundaryARN, !Ref InstanceRolePermissionsBoundaryARN, !Ref "AWS::NoValue" ] - ManagedPolicyArns: !If - - HasManagedPolicies - # Support multiple policies to attach by merging the values together and splitting on ',' - - !Split - - ',' - # Join will skip over AWS::NoValue values - - !Join - - ',' - - - !If - - UseECR - - !FindInMap [ ECRManagedPolicy, !Ref ECRAccessPolicy, 'Policy' ] - - !Ref 'AWS::NoValue' - # This may support multiple values of its own (separated by commas) - - !If - - UseManagedPolicyARN - - !Join [ ',', !Ref ManagedPolicyARN ] - - !Ref 'AWS::NoValue' - - !Ref 'AWS::NoValue' - Policies: - - !If - - UseCustomerManagedKeyForParameterStore - - PolicyName: DecryptAgentToken - PolicyDocument: - Version: '2012-10-17' - Statement: - - Effect: Allow - Action: - - kms:Decrypt - Resource: !Sub arn:aws:kms:${AWS::Region}:${AWS::AccountId}:key/${BuildkiteAgentTokenParameterStoreKMSKey} - - !Ref 'AWS::NoValue' - - PolicyName: ReadAgentToken - PolicyDocument: - Version: '2012-10-17' - Statement: - - Effect: Allow - Action: ssm:GetParameter - Resource: - !Sub - - arn:aws:ssm:${AWS::Region}:${AWS::AccountId}:parameter${ParameterPath} - - ParameterPath: !If [ UseCustomerManagedParameterPath, !Ref BuildkiteAgentTokenParameterStorePath, !Ref BuildkiteAgentTokenParameter ] - AssumeRolePolicyDocument: - Statement: - - Effect: Allow - Principal: - Service: [ autoscaling.amazonaws.com, ec2.amazonaws.com ] - Action: sts:AssumeRole - Path: / - - IAMPolicies: - Type: AWS::IAM::Policy - Properties: - PolicyName: InstancePolicy - PolicyDocument: - Statement: - - !If - - HasSecretsBucket - - Sid: SecretsBucket - Effect: Allow - Action: - - s3:Get* - - s3:List* - Resource: - - !Sub - - "arn:aws:s3:::${Bucket}/*" - - Bucket: !If [ CreateSecretsBucket, !Ref ManagedSecretsBucket, !Ref SecretsBucket ] - - !Sub - - "arn:aws:s3:::${Bucket}" - - Bucket: !If [ CreateSecretsBucket, !Ref ManagedSecretsBucket, !Ref SecretsBucket ] - - !Ref "AWS::NoValue" - - !If - - UseArtifactsBucket - - Sid: ArtifactsBucket - Effect: Allow - Action: - - s3:GetObject - - s3:GetObjectAcl - - s3:GetObjectVersion - - s3:GetObjectVersionAcl - - s3:ListBucket - - s3:PutObject - - s3:PutObjectAcl - - s3:PutObjectVersionAcl - Resource: - - !Sub "arn:aws:s3:::${ArtifactsBucket}/*" - - !Sub "arn:aws:s3:::${ArtifactsBucket}" - - !Ref "AWS::NoValue" - - Effect: Allow - Action: - - autoscaling:DescribeAutoScalingInstances - - cloudwatch:PutMetricData - - cloudformation:DescribeStackResource - - ec2:DescribeTags - Resource: "*" - - Sid: TerminateInstance - Effect: Allow - Action: - - autoscaling:SetInstanceHealth - - autoscaling:TerminateInstanceInAutoScalingGroup - Resource: !Sub arn:${AWS::Partition}:autoscaling:${AWS::Region}:${AWS::AccountId}:autoScalingGroup:*:autoScalingGroupName/${AWS::StackName}-AgentAutoScaleGroup-* - - Sid: Logging - Effect: Allow - Action: - - logs:CreateLogGroup - - logs:CreateLogStream - - logs:PutLogEvents - - logs:DescribeLogGroups - - logs:DescribeLogStreams - Resource: "*" - - Sid: Ssm - Effect: Allow - Action: - - ssm:DescribeInstanceProperties - - ssm:ListAssociations - - ssm:PutInventory - - ssm:UpdateInstanceInformation - - ssmmessages:CreateControlChannel - - ssmmessages:CreateDataChannel - - ssmmessages:OpenControlChannel - - ssmmessages:OpenDataChannel - - ec2messages:AcknowledgeMessage - - ec2messages:DeleteMessage - - ec2messages:FailMessage - - ec2messages:GetEndpoint - - ec2messages:GetMessages - - ec2messages:SendRepl - Resource: "*" - Roles: - - !Ref IAMRole - - ManagedSecretsLoggingBucket: - Type: AWS::S3::Bucket - Condition: CreateSecretsBucket - DeletionPolicy: Retain - Properties: - AccessControl: LogDeliveryWrite - Tags: - - !If - - UseCostAllocationTags - - Key: !Ref CostAllocationTagName - Value: !Ref CostAllocationTagValue - - !Ref "AWS::NoValue" - - ManagedSecretsBucket: - Type: AWS::S3::Bucket - Condition: CreateSecretsBucket - DeletionPolicy: Retain - Properties: - LoggingConfiguration: - DestinationBucketName: !Ref ManagedSecretsLoggingBucket - VersioningConfiguration: - Status: Enabled - Tags: - - !If - - UseCostAllocationTags - - Key: !Ref CostAllocationTagName - Value: !Ref CostAllocationTagValue - - !Ref "AWS::NoValue" - - ImageIdParameterStack: - Type: AWS::CloudFormation::Stack - Condition: HasImageIdParameter - Properties: - TemplateURL: https://s3.amazonaws.com/buildkite-aws-stack/ssm-ami/releases/0.1.0.yml - Parameters: - AmiParameterPath: !Ref ImageIdParameter - - AgentLaunchTemplate: - Type: "AWS::EC2::LaunchTemplate" - Properties: - LaunchTemplateData: - NetworkInterfaces: - - DeviceIndex: 0 - AssociatePublicIpAddress: { Ref: AssociatePublicIpAddress } - Groups: !Split [ ",", !If [ "CreateSecurityGroup", !Ref SecurityGroup, !Ref SecurityGroupId ] ] - KeyName: !If [ "HasKeyName", !Ref KeyName, !Ref 'AWS::NoValue' ] - IamInstanceProfile: - Arn: !GetAtt "IAMInstanceProfile.Arn" - InstanceType: !Select [ "0", !Split [ ",", !Join [ ",", [ !Ref InstanceType, "", "", "" ] ] ] ] - MetadataOptions: - HttpTokens: !Ref IMDSv2Tokens - # Allow containers using a Docker network on the host to receive IDMSv2 responses - HttpPutResponseHopLimit: 2 - InstanceMetadataTags: enabled - Monitoring: - Enabled: !Ref EnableDetailedMonitoring - ImageId: !If - - HasImageId - - !Ref ImageId - - !If - - HasImageIdParameter - - !GetAtt ImageIdParameterStack.Outputs.ImageId - - !If - - UseWindowsAgents - - !FindInMap - - AWSRegion2AMI - - !Ref 'AWS::Region' - - 'windows' - - !If - - UsingArmInstances - - !FindInMap - - AWSRegion2AMI - - !Ref 'AWS::Region' - - 'linuxarm64' - - !FindInMap - - AWSRegion2AMI - - !Ref 'AWS::Region' - - 'linuxamd64' - BlockDeviceMappings: - - DeviceName: !If [ UseDefaultRootVolumeName, !If [ UseWindowsAgents, /dev/sda1, /dev/xvda ], !Ref RootVolumeName ] - Ebs: { VolumeSize: !Ref RootVolumeSize, VolumeType: !Ref RootVolumeType } - TagSpecifications: - - ResourceType: instance - Tags: - - Key: Role - Value: buildkite-agent - - Key: Name - Value: buildkite-agent - - Key: BuildkiteAgentRelease - Value: !Ref BuildkiteAgentRelease - - Key: BuildkiteQueue - Value: !Ref BuildkiteQueue - - !If - - UseCostAllocationTags - - Key: !Ref CostAllocationTagName - Value: !Ref CostAllocationTagValue - - !Ref "AWS::NoValue" - UserData: - Fn::Base64: !If - - UseWindowsAgents - - !Sub - - | - - $Env:DOCKER_USERNS_REMAP="${EnableDockerUserNamespaceRemap}" - $Env:DOCKER_EXPERIMENTAL="${EnableDockerExperimental}" - powershell -file C:\buildkite-agent\bin\bk-configure-docker.ps1 >> C:\buildkite-agent\elastic-stack.log - - $Env:BUILDKITE_STACK_NAME="${AWS::StackName}" - $Env:BUILDKITE_STACK_VERSION="v5.10.0" - $Env:BUILDKITE_SCALE_IN_IDLE_PERIOD="${ScaleInIdlePeriod}" - $Env:BUILDKITE_SECRETS_BUCKET="${LocalSecretsBucket}" - $Env:BUILDKITE_SECRETS_BUCKET_REGION="${LocalSecretsBucketRegion}" - $Env:BUILDKITE_AGENT_TOKEN_PATH="${AgentTokenPath}" - $Env:BUILDKITE_AGENTS_PER_INSTANCE="${AgentsPerInstance}" - $Env:BUILDKITE_AGENT_TAGS="${BuildkiteAgentTags}" - $Env:BUILDKITE_AGENT_TIMESTAMP_LINES="${BuildkiteAgentTimestampLines}" - $Env:BUILDKITE_AGENT_EXPERIMENTS="${BuildkiteAgentExperiments}" - $Env:BUILDKITE_AGENT_RELEASE="${BuildkiteAgentRelease}" - $Env:BUILDKITE_QUEUE="${BuildkiteQueue}" - $Env:BUILDKITE_AGENT_ENABLE_GIT_MIRRORS_EXPERIMENT="${EnableAgentGitMirrorsExperiment}" - $Env:BUILDKITE_ELASTIC_BOOTSTRAP_SCRIPT="${BootstrapScriptUrl}" - $Env:BUILDKITE_AUTHORIZED_USERS_URL="${AuthorizedUsersUrl}" - $Env:BUILDKITE_ECR_POLICY="${ECRAccessPolicy}" - $Env:BUILDKITE_TERMINATE_INSTANCE_AFTER_JOB="${BuildkiteTerminateInstanceAfterJob}" - $Env:BUILDKITE_ADDITIONAL_SUDO_PERMISSIONS="${BuildkiteAdditionalSudoPermissions}" - $Env:BUILDKITE_WINDOWS_ADMINISTRATOR="${BuildkiteWindowsAdministrator}" - $Env:AWS_DEFAULT_REGION="${AWS::Region}" - $Env:SECRETS_PLUGIN_ENABLED="${EnableSecretsPlugin}" - $Env:ECR_PLUGIN_ENABLED="${EnableECRPlugin}" - $Env:DOCKER_LOGIN_PLUGIN_ENABLED="${EnableDockerLoginPlugin}" - $Env:AWS_REGION="${AWS::Region}" - powershell -file C:\buildkite-agent\bin\bk-install-elastic-stack.ps1 >> C:\buildkite-agent\elastic-stack.log - - - { - LocalSecretsBucket: !If [ CreateSecretsBucket, !Ref ManagedSecretsBucket, !Ref SecretsBucket ], - LocalSecretsBucketRegion: !If [ CreateSecretsBucket, !Ref "AWS::Region", !Ref SecretsBucketRegion ], - AgentTokenPath: !If [ UseCustomerManagedParameterPath, !Ref BuildkiteAgentTokenParameterStorePath, !Ref BuildkiteAgentTokenParameter ], - } - - !Sub - - | - Content-Type: multipart/mixed; boundary="==BOUNDARY==" - MIME-Version: 1.0 - --==BOUNDARY== - Content-Type: text/cloud-boothook; charset="us-ascii" - BUILDKITE_ENABLE_INSTANCE_STORAGE="${EnableInstanceStorage}" \ - /usr/local/bin/bk-mount-instance-storage.sh - --==BOUNDARY== - Content-Type: text/cloud-boothook; charset="us-ascii" - DOCKER_USERNS_REMAP=${EnableDockerUserNamespaceRemap} \ - DOCKER_EXPERIMENTAL=${EnableDockerExperimental} \ - BUILDKITE_ENABLE_INSTANCE_STORAGE="${EnableInstanceStorage}" \ - /usr/local/bin/bk-configure-docker.sh - --==BOUNDARY== - Content-Type: text/x-shellscript; charset="us-ascii" - #!/bin/bash -v - BUILDKITE_STACK_NAME="${AWS::StackName}" \ - BUILDKITE_STACK_VERSION="v5.10.0" \ - BUILDKITE_SCALE_IN_IDLE_PERIOD="${ScaleInIdlePeriod}" \ - BUILDKITE_SECRETS_BUCKET="${LocalSecretsBucket}" \ - BUILDKITE_SECRETS_BUCKET_REGION="${LocalSecretsBucketRegion}" \ - BUILDKITE_AGENT_TOKEN_PATH="${AgentTokenPath}" \ - BUILDKITE_AGENTS_PER_INSTANCE="${AgentsPerInstance}" \ - BUILDKITE_AGENT_TAGS="${BuildkiteAgentTags}" \ - BUILDKITE_AGENT_TIMESTAMP_LINES="${BuildkiteAgentTimestampLines}" \ - BUILDKITE_AGENT_EXPERIMENTS="${BuildkiteAgentExperiments}" \ - BUILDKITE_AGENT_RELEASE="${BuildkiteAgentRelease}" \ - BUILDKITE_QUEUE="${BuildkiteQueue}" \ - BUILDKITE_AGENT_ENABLE_GIT_MIRRORS_EXPERIMENT="${EnableAgentGitMirrorsExperiment}" \ - BUILDKITE_ELASTIC_BOOTSTRAP_SCRIPT="${BootstrapScriptUrl}" \ - BUILDKITE_ENABLE_INSTANCE_STORAGE="${EnableInstanceStorage}" \ - BUILDKITE_AUTHORIZED_USERS_URL="${AuthorizedUsersUrl}" \ - BUILDKITE_ECR_POLICY="${ECRAccessPolicy}" \ - BUILDKITE_TERMINATE_INSTANCE_AFTER_JOB="${BuildkiteTerminateInstanceAfterJob}" \ - BUILDKITE_ADDITIONAL_SUDO_PERMISSIONS="${BuildkiteAdditionalSudoPermissions}" \ - AWS_DEFAULT_REGION="${AWS::Region}" \ - SECRETS_PLUGIN_ENABLED="${EnableSecretsPlugin}" \ - ECR_PLUGIN_ENABLED="${EnableECRPlugin}" \ - DOCKER_LOGIN_PLUGIN_ENABLED="${EnableDockerLoginPlugin}" \ - DOCKER_EXPERIMENTAL="${EnableDockerExperimental}" \ - AWS_REGION="${AWS::Region}" \ - /usr/local/bin/bk-install-elastic-stack.sh - --==BOUNDARY==-- - - { - LocalSecretsBucket: !If [ CreateSecretsBucket, !Ref ManagedSecretsBucket, !Ref SecretsBucket ], - LocalSecretsBucketRegion: !If [ CreateSecretsBucket, !Ref "AWS::Region", !Ref SecretsBucketRegion ], - AgentTokenPath: !If [ UseCustomerManagedParameterPath, !Ref BuildkiteAgentTokenParameterStorePath, !Ref BuildkiteAgentTokenParameter ], - } - - AgentAutoScaleGroup: - Type: AWS::AutoScaling::AutoScalingGroup - DependsOn: - - IAMPolicies - - VpcComplete - Properties: - VPCZoneIdentifier: !If [ "CreateVpcResources", [ !Ref Subnet0, !Ref Subnet1 ], !Ref Subnets ] - MixedInstancesPolicy: - InstancesDistribution: - OnDemandPercentageAboveBaseCapacity: !If [ SpotPriceSet, 0, !Ref OnDemandPercentage ] - SpotAllocationStrategy: capacity-optimized - SpotMaxPrice: !If [SpotPriceSet, !Ref SpotPrice, !Ref "AWS::NoValue"] - LaunchTemplate: - LaunchTemplateSpecification: - LaunchTemplateId: !Ref AgentLaunchTemplate - Version: !GetAtt "AgentLaunchTemplate.LatestVersionNumber" - Overrides: - - InstanceType: !Select [ "0", !Split [ ",", !Join [ ",", [ !Ref InstanceType, "", "", "" ] ] ] ] - - !If - - UseInstanceType2 - - InstanceType: !Select [ "1", !Split [ ",", !Join [ ",", [ !Ref InstanceType, "", "", "" ] ] ] ] - - !Ref "AWS::NoValue" - - !If - - UseInstanceType3 - - InstanceType: !Select [ "2", !Split [ ",", !Join [ ",", [ !Ref InstanceType, "", "", "" ] ] ] ] - - !Ref "AWS::NoValue" - - !If - - UseInstanceType4 - - InstanceType: !Select [ "3", !Split [ ",", !Join [ ",", [ !Ref InstanceType, "", "", "" ] ] ] ] - - !Ref "AWS::NoValue" - MinSize: !Ref MinSize - MaxSize: !Ref MaxSize - Cooldown: 60 - MetricsCollection: - - Granularity: 1Minute - Metrics: - - GroupMinSize - - GroupMaxSize - - GroupInServiceInstances - - GroupTerminatingInstances - - GroupPendingInstances - TerminationPolicies: - - OldestLaunchConfiguration - - ClosestToNextInstanceHour - NewInstancesProtectedFromScaleIn: true - CreationPolicy: - ResourceSignal: - Timeout: !If [ UseDefaultInstanceCreationTimeout, !If [ UseWindowsAgents, PT10M, PT5M ], !Ref InstanceCreationTimeout ] - Count: !Ref MinSize - UpdatePolicy: - AutoScalingReplacingUpdate: - WillReplace: true - - AsgProcessSuspenderRole: - Type: AWS::IAM::Role - Properties: - PermissionsBoundary: !If [ SetInstanceRolePermissionsBoundaryARN, !Ref InstanceRolePermissionsBoundaryARN, !Ref "AWS::NoValue" ] - AssumeRolePolicyDocument: - Version: 2012-10-17 - Statement: - - Effect: Allow - Principal: - Service: - - lambda.amazonaws.com - Action: - - sts:AssumeRole - ManagedPolicyArns: - - arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole - Policies: - - PolicyName: AsgProcessModification - PolicyDocument: - Version: 2012-10-17 - Statement: - - Effect: Allow - Action: - - 'autoscaling:SuspendProcesses' - Resource: !Sub arn:${AWS::Partition}:autoscaling:${AWS::Region}:${AWS::AccountId}:autoScalingGroup:*:autoScalingGroupName/${AWS::StackName}-AgentAutoScaleGroup-* - - AzRebalancingSuspenderFunction: - Type: AWS::Lambda::Function - Properties: - Description: 'Disables AZ Rebalancing on the agent ASG' - Code: - ZipFile: | - import cfnresponse - import boto3 - def handler(event, context): - try: - if event['RequestType'] == 'Delete': - cfnresponse.send(event, context, cfnresponse.SUCCESS, {}, "CustomResourcePhysicalID") - else: - client = boto3.client('autoscaling') - props = event['ResourceProperties'] - response = client.suspend_processes(AutoScalingGroupName=props['AutoScalingGroupName'], ScalingProcesses=['AZRebalance']) - cfnresponse.send(event, context, cfnresponse.SUCCESS, {}, "CustomResourcePhysicalID") - except BaseException as err: - print('ERROR: ', err) - cfnresponse.send(event, context, cfnresponse.FAILED, {}, "CustomResourcePhysicalID") - Handler: index.handler - Role: !GetAtt AsgProcessSuspenderRole.Arn - Runtime: 'python3.7' - - AzRebalancingSuspender: - Type: AWS::CloudFormation::CustomResource - Version: 1.0 - Properties: - ServiceToken: !GetAtt AzRebalancingSuspenderFunction.Arn - AutoScalingGroupName: !Ref AgentAutoScaleGroup - - SecurityGroup: - Type: AWS::EC2::SecurityGroup - Condition: CreateSecurityGroup - Properties: - GroupDescription: Enable access to agents - VpcId: !If [ "CreateVpcResources", !Ref Vpc, !Ref VpcId ] - Tags: - - Key: Name - Value: !Ref 'AWS::StackName' - - SecurityGroupSshIngress: - Condition: EnableSshIngress - Type: AWS::EC2::SecurityGroupIngress - Properties: - GroupId: !GetAtt SecurityGroup.GroupId - IpProtocol: tcp - FromPort: 22 - ToPort: 22 - CidrIp: 0.0.0.0/0 - - Autoscaling: - Type: AWS::Serverless::Application - Condition: HasVariableSize - Properties: - Location: - ApplicationId: arn:aws:serverlessrepo:us-east-1:172840064832:applications/buildkite-agent-scaler - SemanticVersion: '1.3.1' - Parameters: - BuildkiteAgentTokenParameter: !If [ UseCustomerManagedParameterPath, !Ref BuildkiteAgentTokenParameterStorePath, !Ref BuildkiteAgentTokenParameter ] - BuildkiteAgentTokenParameterStoreKMSKey: !If [ UseCustomerManagedKeyForParameterStore, !Ref BuildkiteAgentTokenParameterStoreKMSKey, "" ] - RolePermissionsBoundaryARN: !If [ SetInstanceRolePermissionsBoundaryARN, !Ref InstanceRolePermissionsBoundaryARN, "" ] - BuildkiteQueue: !Ref BuildkiteQueue - AgentsPerInstance: !Ref AgentsPerInstance - MinSize: !Ref MinSize - MaxSize: !Ref MaxSize - AgentAutoScaleGroup: !Ref AgentAutoScaleGroup - ScaleOutFactor: !Ref ScaleOutFactor - ScaleOutForWaitingJobs: !Ref ScaleOutForWaitingJobs diff --git a/terraform/buildkite-v5.16.1.yml b/terraform/buildkite-v5.16.1.yml index 3b17300..68106d6 100644 --- a/terraform/buildkite-v5.16.1.yml +++ b/terraform/buildkite-v5.16.1.yml @@ -1100,7 +1100,7 @@ Resources: $Env:DOCKER_USERNS_REMAP="${EnableDockerUserNamespaceRemap}" $Env:DOCKER_EXPERIMENTAL="${EnableDockerExperimental}" powershell -file C:\buildkite-agent\bin\bk-configure-docker.ps1 >> C:\buildkite-agent\elastic-stack.log - + $Env:BUILDKITE_STACK_NAME="${AWS::StackName}" $Env:BUILDKITE_STACK_VERSION="v5.16.1" $Env:BUILDKITE_SCALE_IN_IDLE_PERIOD="${ScaleInIdlePeriod}" @@ -1333,4 +1333,4 @@ Resources: MaxSize: !Ref MaxSize AgentAutoScaleGroup: !Ref AgentAutoScaleGroup ScaleOutFactor: !Ref ScaleOutFactor - ScaleOutForWaitingJobs: !Ref ScaleOutForWaitingJobs \ No newline at end of file + ScaleOutForWaitingJobs: !Ref ScaleOutForWaitingJobs diff --git a/terraform/buildkite-v5.8.0.yml b/terraform/buildkite-v5.8.0.yml deleted file mode 100644 index e0b42df..0000000 --- a/terraform/buildkite-v5.8.0.yml +++ /dev/null @@ -1,1219 +0,0 @@ ---- -AWSTemplateFormatVersion: "2010-09-09" -Description: "Buildkite stack v5.8.0" - -# The Buildkite Elastic CI Stack for AWS gives you a private, -# autoscaling Buildkite Agent cluster. Use it to parallelize -# large test suites across thousands of nodes, run tests and -# deployments for Linux or Windows based services and apps, -# or run AWS ops tasks. -# -# To gain a better understanding of how Elastic CI Stack works -# and how to use it most effectively and securely, check out -# the following resources: -# -# * Elastic CI Stack for AWS Overview: https://buildkite.com/docs/agent/v3/elastic_ci_aws -# * Elastic CI Stack for AWS Tutorial: https://buildkite.com/docs/tutorials/elastic-ci-stack-aws -# * Running Buildkite Agent on AWS: https://buildkite.com/docs/agent/v3/aws -# * GitHub Repo for Elastic CI Stack: https://github.com/buildkite/elastic-ci-stack-for-aws -# * Template Parameters for Elastic CI Stack for AWS: https://buildkite.com/docs/agent/v3/elastic-ci-aws/parameters -# * Using AWS Secrets Manager: https://buildkite.com/docs/agent/v3/aws/secrets-manager -# * VPC Design: https://buildkite.com/docs/agent/v3/aws/vpc -# * CloudFormation Service Role: https://buildkite.com/docs/agent/v3/elastic-ci-aws/cloudformation-service-role - -Transform: AWS::Serverless-2016-10-31 - -Metadata: - AWS::CloudFormation::Interface: - ParameterGroups: - - Label: - default: Buildkite Configuration - Parameters: - - BuildkiteAgentTokenParameterStorePath - - BuildkiteAgentTokenParameterStoreKMSKey - - BuildkiteAgentToken - - BuildkiteQueue - - - Label: - default: Advanced Buildkite Configuration - Parameters: - - BuildkiteAgentRelease - - BuildkiteAgentTags - - BuildkiteAgentTimestampLines - - BuildkiteAgentExperiments - - BuildkiteTerminateInstanceAfterJob - - BuildkiteAdditionalSudoPermissions - - BuildkiteWindowsAdministrator - - - Label: - default: Network Configuration - Parameters: - - VpcId - - Subnets - - AvailabilityZones - - SecurityGroupId - - AssociatePublicIpAddress - - - Label: - default: Instance Configuration - Parameters: - - ImageId - - ImageIdParameter - - InstanceType - - EnableInstanceStorage - - AgentsPerInstance - - KeyName - - SpotPrice - - SecretsBucket - - SecretsBucketRegion - - ArtifactsBucket - - AuthorizedUsersUrl - - BootstrapScriptUrl - - RootVolumeSize - - RootVolumeName - - RootVolumeType - - ManagedPolicyARN - - InstanceRoleName - - IMDSv2Tokens - - - Label: - default: Auto-scaling Configuration - Parameters: - - MinSize - - MaxSize - - OnDemandPercentage - - ScaleOutFactor - - ScaleInIdlePeriod - - ScaleOutForWaitingJobs - - InstanceCreationTimeout - - - Label: - default: Cost Allocation Configuration - Parameters: - - EnableCostAllocationTags - - CostAllocationTagName - - CostAllocationTagValue - - - Label: - default: Docker Daemon Configuration - Parameters: - - EnableDockerUserNamespaceRemap - - EnableDockerExperimental - - - Label: - default: Docker Registry Configuration - Parameters: - - ECRAccessPolicy - - - Label: - default: Plugin Configuration - Parameters: - - EnableSecretsPlugin - - EnableECRPlugin - - EnableDockerLoginPlugin - -Parameters: - KeyName: - Description: Optional - SSH keypair used to access the buildkite instances via ec2_user, setting this will enable SSH ingress - Type: String - Default: "" - - BuildkiteAgentRelease: - Type: String - AllowedValues: - - stable - - beta - - edge - Default: "stable" - - BuildkiteAgentToken: - Description: Buildkite agent registration token. Or, preload it into SSM Parameter Store and use BuildkiteAgentTokenParameterStorePath for secure environments. - Type: String - NoEcho: true - Default: "" - - BuildkiteAgentTokenParameterStorePath: - Description: Existing SSM Parameter Store path to the Buildkite agent registration token (takes precedence over BuildkiteAgentToken). Expects a leading slash ('/'). - Type: String - Default: "" - AllowedPattern: "^$|^/[a-zA-Z0-9_.\\-/]+$" - ConstraintDescription: "Expects a leading forward slash" - - BuildkiteAgentTokenParameterStoreKMSKey: - Description: AWS KMS key ID used to encrypt the SSM parameter (if encrypted) - Type: String - Default: "" - - BuildkiteAgentTags: - Description: Additional tags separated by commas to provide to the agent. E.g os=linux,llamas=always - Type: String - Default: "" - - BuildkiteAgentTimestampLines: - Description: Set to true to prepend timestamps to every line of output - Type: String - AllowedValues: - - "true" - - "false" - Default: "false" - - BuildkiteAgentExperiments: - Description: Agent experiments to enable, comma delimited. See https://github.com/buildkite/agent/blob/master/EXPERIMENTS.md. - Type: String - Default: "" - - BuildkiteTerminateInstanceAfterJob: - Description: Set to "true" to terminate the instance after a job has completed. - Type: String - AllowedValues: - - "true" - - "false" - Default: "false" - - BuildkiteAdditionalSudoPermissions: - Description: Optional - Comma separated list of commands to allow the buildkite-agent user to run using sudo. - Type: String - Default: "" - - BuildkiteWindowsAdministrator: - Description: Set to "true" to add the local "buildkite-agent" user account to the local Windows Administrator group. - Type: String - AllowedValues: - - "true" - - "false" - Default: "true" - - BuildkiteQueue: - Description: Queue name that agents will use, targeted in pipeline steps using "queue={value}" - Type: String - Default: default - MinLength: 1 - - AgentsPerInstance: - Description: Number of Buildkite agents to run on each instance - Type: Number - Default: 1 - MinValue: 1 - - SecretsBucket: - Description: Optional - Name of an existing S3 bucket containing pipeline secrets (Created if left blank) - Type: String - Default: "" - - SecretsBucketRegion: - Description: Optional - Region for the SecretsBucket. If blank the bucket's region is dynamically discovered. - Type: String - Default: "" - - ArtifactsBucket: - Description: Optional - Name of an existing S3 bucket for build artifact storage - Type: String - Default: "" - - BootstrapScriptUrl: - Description: Optional - HTTPS or S3 URL to run on each instance during boot - Type: String - Default: "" - - AuthorizedUsersUrl: - Description: Optional - HTTPS or S3 URL to periodically download ssh authorized_keys from, setting this will enable SSH ingress. authorized_keys are applied to ec2_user - Type: String - Default: "" - - VpcId: - Type: String - Description: Optional - Id of an existing VPC to launch instances into. Leave blank to have a new VPC created - Default: "" - - Subnets: - Type: CommaDelimitedList - Description: Optional - Comma separated list of two existing VPC subnet ids where EC2 instances will run. Required if setting VpcId. - Default: "" - - AvailabilityZones: - Type: CommaDelimitedList - Description: Optional - Comma separated list of AZs that subnets are created in (if Subnets parameter is not specified) - Default: "" - - InstanceType: - Description: Instance type. Comma-separated list with 1-4 instance types. The order is a prioritized preference for launching OnDemand instances, and a non-prioritized list of types to consider for Spot Instances (where used). - Type: String - Default: t3.large - MinLength: 1 - AllowedPattern: "^[\\w\\.]+(,[\\w\\.]*){0,3}$" - ConstraintDescription: "must contain 1-4 instance types separated by commas. No space before/after the comma." - - SpotPrice: - Description: Maximum spot price to use for the instances, in instance cost per hour. Values >0 will result in 100% of instances being spot. 0 means only use normal (non-spot) instances. This parameter is deprecated - we recommend setting to 0 and using OnDemandPercentage to opt into spot instances. - Type: String - Default: 0 - - MaxSize: - Description: Maximum number of instances - Type: Number - Default: 10 - MinValue: 1 - - MinSize: - Description: Minimum number of instances - Type: Number - Default: 0 - - OnDemandPercentage: - Description: Percentage of total instances that should launch as OnDemand. Default is 100% OnDemand - reduce this to use some Spot Instances when they're available and cheaper than the OnDemand price. A value of 70 means 70% OnDemand and 30% Spot Instances. - Type: Number - Default: 100 - MinValue: 0 - MaxValue: 100 - - ScaleOutFactor: - Description: A decimal factor to apply to scale out changes to speed up or slow down scale-out - Type: Number - Default: 1.0 - - ScaleInIdlePeriod: - Description: Number of seconds an agent must be idle before terminating - Type: Number - Default: 600 - - ScaleOutForWaitingJobs: - Type: String - Description: Whether to scale-out for steps behind wait steps. Make sure you have a long enough idle period! - AllowedValues: - - "true" - - "false" - Default: "false" - - InstanceCreationTimeout: - Description: Timeout period for Autoscaling Group Creation Policy - Type: String - Default: "" - - RootVolumeSize: - Description: Size of each instance's root EBS volume (in GB) - Type: Number - Default: 250 - MinValue: 10 - - RootVolumeName: - Description: Name of the root block device for your AMI - Type: String - Default: "" - - RootVolumeType: - Description: Type of root volume to use - Type: String - Default: "gp3" - - SecurityGroupId: - Type: String - Description: Optional - Comma separated list of security group ids to assign to instances - Default: "" - - ImageId: - Type: String - Description: Optional - Custom AMI to use for instances (must be based on the stack's AMI) - Default: "" - - ImageIdParameter: - Type: String - Description: Optional - Custom AMI SSM Parameter to use for instances (must be based on the stack's AMI) - Default: "" - - ManagedPolicyARN: - Type: CommaDelimitedList - Description: Optional - Comma separated list of managed IAM policy ARNs to attach to the instance role - Default: "" - - IMDSv2Tokens: - Type: String - Description: Whether IMDSv2 tokens must be used for the Instance Metadata Service. - AllowedValues: - - optional - - required - Default: optional - - InstanceRoleName: - Type: String - Description: Optional - A name for the IAM Role attached to the Instance Profile - Default: "" - - InstanceRolePermissionsBoundaryARN: - Type: String - Description: The ARN of the policy used to set the permissions boundary for the role. - Default: "" - - InstanceOperatingSystem: - Type: String - Description: The operating system to run on the instances - AllowedValues: - - linux - - windows - Default: "linux" - - ECRAccessPolicy: - Type: String - Description: ECR access policy to give container instances - AllowedValues: - - none - - readonly - - poweruser - - full - Default: "none" - - AssociatePublicIpAddress: - Type: String - Description: Associate instances with public IP addresses - AllowedValues: - - "true" - - "false" - Default: "true" - - EnableSecretsPlugin: - Type: String - Description: Enables s3-secrets plugin for all pipelines - AllowedValues: - - "true" - - "false" - Default: "true" - - EnableECRPlugin: - Type: String - Description: Enables ecr plugin for all pipelines - AllowedValues: - - "true" - - "false" - Default: "true" - - EnableDockerLoginPlugin: - Type: String - Description: Enables docker-login plugin for all pipelines - AllowedValues: - - "true" - - "false" - Default: "true" - - EnableDockerUserNamespaceRemap: - Type: String - Description: Enables Docker user namespace remapping so docker runs as buildkite-agent - AllowedValues: - - "true" - - "false" - Default: "true" - - EnableDockerExperimental: - Type: String - Description: Enables Docker experimental features - AllowedValues: - - "true" - - "false" - Default: "false" - - EnableInstanceStorage: - Type: String - Description: Mount available NVMe Instance Storage at /mnt/ephemeral - AllowedValues: - - "true" - - "false" - Default: "false" - - EnableCostAllocationTags: - Type: String - Description: Enables AWS Cost Allocation tags for all resources in the stack. See https://docs.aws.amazon.com/awsaccountbilling/latest/aboutv2/cost-alloc-tags.html - AllowedValues: - - "true" - - "false" - Default: "false" - - CostAllocationTagName: - Type: String - Description: The name of the Cost Allocation Tag used for billing purposes - Default: "CreatedBy" - - CostAllocationTagValue: - Type: String - Description: The value of the Cost Allocation Tag used for billing purposes - Default: "buildkite-elastic-ci-stack-for-aws" - - EnableAgentGitMirrorsExperiment: - Type: String - Description: Enables the git-mirrors experiment in the agent - AllowedValues: - - "true" - - "false" - Default: "false" - -Rules: - HasToken: - Assertions: - - Assert: - !Or - - !Not - - !Equals - - !Ref BuildkiteAgentToken - - "" - - !Not - - !Equals - - !Ref BuildkiteAgentTokenParameterStorePath - - "" - AssertDescription: "You must provide BuildkiteAgentToken or BuildkiteAgentTokenParameterStorePath" - -Outputs: - VpcId: - Value: - !If [ CreateVpcResources, !Ref Vpc, !Ref VpcId ] - Export: - Name: !Sub '${AWS::StackName}-VpcId' - - ManagedSecretsBucket: - Value: - !If [ CreateSecretsBucket, !Ref ManagedSecretsBucket, "Undefined" ] - Export: - Name: !Sub '${AWS::StackName}-ManagedSecretsBucket' - - ManagedSecretsLoggingBucket: - Value: - !If [ CreateSecretsBucket, !Ref ManagedSecretsLoggingBucket, "Undefined" ] - Export: - Name: !Sub '${AWS::StackName}-ManagedSecretsLoggingBucket' - - AutoScalingGroupName: - Value: !Ref AgentAutoScaleGroup - Export: - Name: !Sub '${AWS::StackName}-AutoScalingGroupName' - - InstanceRoleName: - Value: !Ref IAMRole - Export: - Name: !Sub '${AWS::StackName}-InstanceRoleName' - -Conditions: - SpotPriceSet: - !Not [ !Equals [ !Ref SpotPrice, 0 ] ] - - CreateVpcResources: - !Equals [ !Ref VpcId, "" ] - - CreateSecurityGroup: - !Equals [ !Ref SecurityGroupId, "" ] - - CreateSecretsBucket: - !And - - !Equals [ !Ref EnableSecretsPlugin, "true"] - - !Equals [ !Ref SecretsBucket, "" ] - - SetInstanceRoleName: - !Not [ !Equals [ !Ref InstanceRoleName, "" ] ] - - SetInstanceRolePermissionsBoundaryARN: - !Not [ !Equals [ !Ref InstanceRolePermissionsBoundaryARN, "" ] ] - - UseSpecifiedSecretsBucket: - !Not [ !Equals [ !Ref SecretsBucket, "" ] ] - - HasSecretsBucket: - !Or [ !Condition CreateSecretsBucket, !Condition UseSpecifiedSecretsBucket ] - - UseSpecifiedAvailabilityZones: - !Not [ !Equals [ !Join [ "", !Ref AvailabilityZones ], "" ] ] - - UseArtifactsBucket: - !Not [ !Equals [ !Ref ArtifactsBucket, "" ] ] - - HasImageId: - !Not [ !Equals [ !Ref ImageId, "" ] ] - HasImageIdParameter: - !Not [ !Equals [ !Ref ImageIdParameter, "" ] ] - - UseDefaultInstanceCreationTimeout: - !Equals [ !Ref InstanceCreationTimeout, "" ] - - UseDefaultRootVolumeName: - !Equals [ !Ref RootVolumeName, "" ] - - UseInstanceType2: - !Not [ !Equals [ !Select [ "1", !Split [ ",", !Join [ ",", [ !Ref InstanceType, "", "", "" ] ] ] ], ""] ] - - UseInstanceType3: - !Not [ !Equals [ !Select [ "2", !Split [ ",", !Join [ ",", [ !Ref InstanceType, "", "", "" ] ] ] ], ""] ] - - UseInstanceType4: - !Not [ !Equals [ !Select [ "3", !Split [ ",", !Join [ ",", [ !Ref InstanceType, "", "", "" ] ] ] ], ""] ] - - UseManagedPolicyARN: - !Not [ !Equals [ !Join [ "", !Ref ManagedPolicyARN ], "" ] ] - - UseECR: - !Not [ !Equals [ !Ref ECRAccessPolicy, "none" ] ] - - UseCustomerManagedParameterPath: - !Not [ !Equals [ !Ref BuildkiteAgentTokenParameterStorePath, "" ] ] - UseCustomerManagedKeyForParameterStore: - !Not [ !Equals [ !Ref BuildkiteAgentTokenParameterStoreKMSKey, "" ] ] - CreateAgentTokenParameter: - !Equals [ !Ref BuildkiteAgentTokenParameterStorePath, "" ] - - HasVariableSize: - !Not [ !Equals [ !Ref MaxSize, !Ref MinSize ] ] - - UseCostAllocationTags: - !Equals [ !Ref EnableCostAllocationTags, "true" ] - - HasKeyName: - !Not [ !Equals [ !Ref KeyName, "" ] ] - - EnableSshIngress: - !And - - { Condition : CreateSecurityGroup } - # Enable ingress if a key can be specified another way - - !Or - - { Condition: HasKeyName } - - !Not [ !Equals [ !Ref AuthorizedUsersUrl, "" ] ] - - # Whether or not there's any managed polices to attach - HasManagedPolicies: - !Or [ { Condition: UseManagedPolicyARN }, { Condition: UseECR } ] - - UseWindowsAgents: - !Equals [ !Ref InstanceOperatingSystem, "windows" ] - - UseLinuxAgents: - !Equals [ !Ref InstanceOperatingSystem, "linux" ] - - # Unfortunately, Cloudformation's !Or intrinsic function only accepts - # between 2 and 10 arguments. To get around this, we're grouping the - # instance families in sub-conditionals. At least this doesn't force us - # into using a Custom Resource. - UsingArmInstances: - !Or - - !Equals [ !Select [ 0, !Split [ ".", !Ref InstanceType ] ], "a1" ] - - !Or - - !Equals [ !Select [ 0, !Split [ ".", !Ref InstanceType ] ], "c6g" ] - - !Equals [ !Select [ 0, !Split [ ".", !Ref InstanceType ] ], "c6gd" ] - - !Equals [ !Select [ 0, !Split [ ".", !Ref InstanceType ] ], "c6gn" ] - - !Equals [ !Select [ 0, !Split [ ".", !Ref InstanceType ] ], "c7g" ] - - !Equals [ !Select [ 0, !Split [ ".", !Ref InstanceType ] ], "g5g" ] - - !Or - - !Equals [ !Select [ 0, !Split [ ".", !Ref InstanceType ] ], "Im4gn" ] - - !Equals [ !Select [ 0, !Split [ ".", !Ref InstanceType ] ], "Is4gen" ] - - !Or - - !Equals [ !Select [ 0, !Split [ ".", !Ref InstanceType ] ], "m6g" ] - - !Equals [ !Select [ 0, !Split [ ".", !Ref InstanceType ] ], "m6gd" ] - - !Or - - !Equals [ !Select [ 0, !Split [ ".", !Ref InstanceType ] ], "r6g" ] - - !Equals [ !Select [ 0, !Split [ ".", !Ref InstanceType ] ], "r6gd" ] - - !Equals [ !Select [ 0, !Split [ ".", !Ref InstanceType ] ], "t4g" ] - - !Equals [ !Select [ 0, !Split [ ".", !Ref InstanceType ] ], "x2gd" ] - -Mappings: - ECRManagedPolicy: - none : { Policy: '' } - readonly : { Policy: 'arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly' } - poweruser : { Policy: 'arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryPowerUser' } - full : { Policy: 'arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryFullAccess' } - - # Generated from Makefile via build/mappings.yml - AWSRegion2AMI: - us-east-1 : { linuxamd64: ami-01667bf01f32d646a, linuxarm64: ami-0a5a50a0fc7d9d433, windows: ami-08bf5be5dc132774a } - us-east-2 : { linuxamd64: ami-064d7e03c816c24a4, linuxarm64: ami-017fc42521d042789, windows: ami-0e7a81b40610d6f24 } - us-west-1 : { linuxamd64: ami-0ef4920e8b2c72948, linuxarm64: ami-06bfb907f1db3eb67, windows: ami-08700eaa3c3b648cf } - us-west-2 : { linuxamd64: ami-035ae8b0ed650b209, linuxarm64: ami-04975db851c3d1417, windows: ami-0e6a3a209764e8bc0 } - af-south-1 : { linuxamd64: ami-09802f9cd41d31544, linuxarm64: ami-042d7dfbe6247a260, windows: ami-0f1d2e023ead42e1b } - ap-east-1 : { linuxamd64: ami-026067f2d2eb09b30, linuxarm64: ami-084cb04a7f5faccc4, windows: ami-0d906a08e26045ad3 } - ap-south-1 : { linuxamd64: ami-08db1022636ee98fc, linuxarm64: ami-085dad57187479931, windows: ami-072e33e4103a83fc9 } - ap-northeast-2 : { linuxamd64: ami-0b9fcea36ae3ca7b0, linuxarm64: ami-04f0bcf1f91a5c6f1, windows: ami-058bab221cb90a562 } - ap-northeast-1 : { linuxamd64: ami-0e9ca3b1ce33fd54d, linuxarm64: ami-025dd23851ee21ba4, windows: ami-099f00ce0343e78fc } - ap-southeast-2 : { linuxamd64: ami-076a6311f31e4730b, linuxarm64: ami-00461bc380f2d468d, windows: ami-017b020f1288fa2da } - ap-southeast-1 : { linuxamd64: ami-03d77260ad960052a, linuxarm64: ami-0c5a12fe2b18a551b, windows: ami-00537a2b7926ac9e6 } - ca-central-1 : { linuxamd64: ami-0d6d2a940d85310fd, linuxarm64: ami-06b7bb1dcc7c7a504, windows: ami-0a46ee45be497a00f } - eu-central-1 : { linuxamd64: ami-0b5fb8fa7db8e87ad, linuxarm64: ami-038e49a8163ccee82, windows: ami-034fe3b2115280ccb } - eu-west-1 : { linuxamd64: ami-02ad95f1546c32df8, linuxarm64: ami-0ce604f641e99165e, windows: ami-070557af241caff2e } - eu-west-2 : { linuxamd64: ami-0d8791b475dad8a12, linuxarm64: ami-08231239f1b483a0c, windows: ami-0c12ae9c741bc5e9e } - eu-south-1 : { linuxamd64: ami-0ec9343d97b69a4c7, linuxarm64: ami-0258bc8c169329747, windows: ami-02979e59a8a6f8d07 } - eu-west-3 : { linuxamd64: ami-0117919d6fb24dcca, linuxarm64: ami-037154cb356780bf2, windows: ami-0449d0c1dd3e494e9 } - eu-north-1 : { linuxamd64: ami-03aa78262fbc7ed30, linuxarm64: ami-09b9c1330698b1d1a, windows: ami-0c495c065b4dd130e } - me-south-1 : { linuxamd64: ami-0649a7e58505e7770, linuxarm64: ami-0e772a8f5a3ccbee4, windows: ami-0204af02669567510 } - sa-east-1 : { linuxamd64: ami-08ed2fa0d4576ab32, linuxarm64: ami-02a8b6834279a1769, windows: ami-09e1ee89b6d4a1d91 } - -Resources: - Vpc: - Type: AWS::EC2::VPC - Condition: CreateVpcResources - Properties: - CidrBlock: 10.0.0.0/16 - InstanceTenancy: default - Tags: - - Key: Name - Value: !Ref 'AWS::StackName' - - Gateway: - Type: AWS::EC2::InternetGateway - Condition: CreateVpcResources - Properties: - Tags: - - Key: Name - Value: !Ref 'AWS::StackName' - - GatewayAttachment: - Type: AWS::EC2::VPCGatewayAttachment - Condition: CreateVpcResources - Properties: - InternetGatewayId: !Ref Gateway - VpcId: !Ref Vpc - - Subnet0: - Type: AWS::EC2::Subnet - Condition: CreateVpcResources - DependsOn: - - GatewayAttachment - Properties: - AvailabilityZone: - !If - - "UseSpecifiedAvailabilityZones" - - !Select [ 0, !Ref AvailabilityZones ] - - !Select [ 0, !GetAZs '' ] - CidrBlock: 10.0.1.0/24 - VpcId: !Ref Vpc - Tags: - - Key: Name - Value: !Ref 'AWS::StackName' - - Subnet1: - Type: AWS::EC2::Subnet - Condition: CreateVpcResources - DependsOn: - - GatewayAttachment - Properties: - AvailabilityZone: - !If - - "UseSpecifiedAvailabilityZones" - - !Select [ 1, !Ref AvailabilityZones ] - - !Select [ 1, !GetAZs '' ] - CidrBlock: 10.0.2.0/24 - VpcId: !Ref Vpc - Tags: - - Key: Name - Value: !Ref 'AWS::StackName' - - Routes: - Type: AWS::EC2::RouteTable - Condition: CreateVpcResources - Properties: - VpcId: !Ref Vpc - Tags: - - Key: Name - Value: !Ref 'AWS::StackName' - - RouteDefault: - Type: AWS::EC2::Route - Condition: CreateVpcResources - DependsOn: - - GatewayAttachment - Properties: - DestinationCidrBlock: 0.0.0.0/0 - GatewayId: !Ref Gateway - RouteTableId: !Ref Routes - - Subnet0Routes: - Type: AWS::EC2::SubnetRouteTableAssociation - Condition: CreateVpcResources - Properties: - SubnetId: !Ref Subnet0 - RouteTableId: !Ref Routes - - Subnet1Routes: - Type: AWS::EC2::SubnetRouteTableAssociation - Condition: CreateVpcResources - Properties: - SubnetId: !Ref Subnet1 - RouteTableId: !Ref Routes - - # A resource that depends on the leaf nodes of the VPC configuration so that - # a strict ordering can be established on teardown. - VpcComplete: - Type: AWS::CloudFormation::WaitConditionHandle - Metadata: - VpcResources: !If - - CreateVpcResources - - [ !Ref RouteDefault, !Ref Subnet0Routes, !Ref Subnet1Routes ] - - !Ref AWS::NoValue - - BuildkiteAgentTokenParameter: - Type: AWS::SSM::Parameter - Condition: CreateAgentTokenParameter - Properties: - Name: !Sub "/${AWS::StackName}/buildkite/agent-token" - Type: String - Value: !Ref BuildkiteAgentToken - - # Allow ec2 instances to assume a role and be granted the IAMPolicies - IAMInstanceProfile: - Type: AWS::IAM::InstanceProfile - Properties: - Path: / - Roles: [ !Ref IAMRole ] - - IAMRole: - Type: AWS::IAM::Role - Properties: - RoleName: !If [ SetInstanceRoleName, !Ref InstanceRoleName, !Sub "${AWS::StackName}-Role" ] - PermissionsBoundary: !If [ SetInstanceRolePermissionsBoundaryARN, !Ref InstanceRolePermissionsBoundaryARN, !Ref "AWS::NoValue" ] - ManagedPolicyArns: !If - - HasManagedPolicies - # Support multiple policies to attach by merging the values together and splitting on ',' - - !Split - - ',' - # Join will skip over AWS::NoValue values - - !Join - - ',' - - - !If - - UseECR - - !FindInMap [ ECRManagedPolicy, !Ref ECRAccessPolicy, 'Policy' ] - - !Ref 'AWS::NoValue' - # This may support multiple values of its own (separated by commas) - - !If - - UseManagedPolicyARN - - !Join [ ',', !Ref ManagedPolicyARN ] - - !Ref 'AWS::NoValue' - - !Ref 'AWS::NoValue' - Policies: - - !If - - UseCustomerManagedKeyForParameterStore - - PolicyName: DecryptAgentToken - PolicyDocument: - Version: '2012-10-17' - Statement: - - Effect: Allow - Action: - - kms:Decrypt - Resource: !Sub arn:aws:kms:${AWS::Region}:${AWS::AccountId}:key/${BuildkiteAgentTokenParameterStoreKMSKey} - - !Ref 'AWS::NoValue' - - PolicyName: ReadAgentToken - PolicyDocument: - Version: '2012-10-17' - Statement: - - Effect: Allow - Action: ssm:GetParameter - Resource: - !Sub - - arn:aws:ssm:${AWS::Region}:${AWS::AccountId}:parameter${ParameterPath} - - ParameterPath: !If [ UseCustomerManagedParameterPath, !Ref BuildkiteAgentTokenParameterStorePath, !Ref BuildkiteAgentTokenParameter ] - AssumeRolePolicyDocument: - Statement: - - Effect: Allow - Principal: - Service: [ autoscaling.amazonaws.com, ec2.amazonaws.com ] - Action: sts:AssumeRole - Path: / - - IAMPolicies: - Type: AWS::IAM::Policy - Properties: - PolicyName: InstancePolicy - PolicyDocument: - Statement: - - !If - - HasSecretsBucket - - Sid: SecretsBucket - Effect: Allow - Action: - - s3:Get* - - s3:List* - Resource: - - !Sub - - "arn:aws:s3:::${Bucket}/*" - - Bucket: !If [ CreateSecretsBucket, !Ref ManagedSecretsBucket, !Ref SecretsBucket ] - - !Sub - - "arn:aws:s3:::${Bucket}" - - Bucket: !If [ CreateSecretsBucket, !Ref ManagedSecretsBucket, !Ref SecretsBucket ] - - !Ref AWS::NoValue - - !If - - UseArtifactsBucket - - Sid: ArtifactsBucket - Effect: Allow - Action: - - s3:GetObject - - s3:GetObjectAcl - - s3:GetObjectVersion - - s3:GetObjectVersionAcl - - s3:ListBucket - - s3:PutObject - - s3:PutObjectAcl - - s3:PutObjectVersionAcl - Resource: - - !Sub "arn:aws:s3:::${ArtifactsBucket}/*" - - !Sub "arn:aws:s3:::${ArtifactsBucket}" - - !Ref AWS::NoValue - - Effect: Allow - Action: - - autoscaling:DescribeAutoScalingInstances - - cloudwatch:PutMetricData - - cloudformation:DescribeStackResource - - ec2:DescribeTags - Resource: "*" - - Sid: TerminateInstance - Effect: Allow - Action: - - autoscaling:SetInstanceHealth - - autoscaling:TerminateInstanceInAutoScalingGroup - Resource: !Sub arn:${AWS::Partition}:autoscaling:${AWS::Region}:${AWS::AccountId}:autoScalingGroup:*:autoScalingGroupName/${AWS::StackName}-AgentAutoScaleGroup-* - - Sid: Logging - Effect: Allow - Action: - - logs:CreateLogGroup - - logs:CreateLogStream - - logs:PutLogEvents - - logs:DescribeLogStreams - Resource: "*" - - Sid: Ssm - Effect: Allow - Action: - - ssm:DescribeInstanceProperties - - ssm:ListAssociations - - ssm:PutInventory - - ssm:UpdateInstanceInformation - - ssmmessages:CreateControlChannel - - ssmmessages:CreateDataChannel - - ssmmessages:OpenControlChannel - - ssmmessages:OpenDataChannel - - ec2messages:AcknowledgeMessage - - ec2messages:DeleteMessage - - ec2messages:FailMessage - - ec2messages:GetEndpoint - - ec2messages:GetMessages - - ec2messages:SendRepl - Resource: "*" - Roles: - - !Ref IAMRole - - ManagedSecretsLoggingBucket: - Type: AWS::S3::Bucket - Condition: CreateSecretsBucket - DeletionPolicy: Retain - Properties: - AccessControl: LogDeliveryWrite - Tags: - - !If - - UseCostAllocationTags - - Key: !Ref CostAllocationTagName - Value: !Ref CostAllocationTagValue - - !Ref "AWS::NoValue" - - ManagedSecretsBucket: - Type: AWS::S3::Bucket - Condition: CreateSecretsBucket - DeletionPolicy: Retain - Properties: - LoggingConfiguration: - DestinationBucketName: !Ref ManagedSecretsLoggingBucket - VersioningConfiguration: - Status: Enabled - Tags: - - !If - - UseCostAllocationTags - - Key: !Ref CostAllocationTagName - Value: !Ref CostAllocationTagValue - - !Ref "AWS::NoValue" - - ImageIdParameterStack: - Type: AWS::CloudFormation::Stack - Condition: HasImageIdParameter - Properties: - TemplateURL: https://s3.amazonaws.com/buildkite-aws-stack/ssm-ami/releases/0.1.0.yml - Parameters: - AmiParameterPath: !Ref ImageIdParameter - - AgentLaunchTemplate: - Type: "AWS::EC2::LaunchTemplate" - Properties: - LaunchTemplateData: - NetworkInterfaces: - - DeviceIndex: 0 - AssociatePublicIpAddress: { Ref: AssociatePublicIpAddress } - Groups: !Split [ ",", !If [ "CreateSecurityGroup", !Ref SecurityGroup, !Ref SecurityGroupId ] ] - KeyName: !If [ "HasKeyName", !Ref KeyName, !Ref 'AWS::NoValue' ] - IamInstanceProfile: - Arn: !GetAtt "IAMInstanceProfile.Arn" - InstanceType: !Select [ "0", !Split [ ",", !Join [ ",", [ !Ref InstanceType, "", "", "" ] ] ] ] - MetadataOptions: - HttpTokens: !Ref IMDSv2Tokens - # Allow containers using a Docker network on the host to receive IDMSv2 responses - HttpPutResponseHopLimit: 2 - ImageId: !If - - HasImageId - - !Ref ImageId - - !If - - HasImageIdParameter - - !GetAtt ImageIdParameterStack.Outputs.ImageId - - !If - - UseWindowsAgents - - !FindInMap - - AWSRegion2AMI - - !Ref 'AWS::Region' - - 'windows' - - !If - - UsingArmInstances - - !FindInMap - - AWSRegion2AMI - - !Ref 'AWS::Region' - - 'linuxarm64' - - !FindInMap - - AWSRegion2AMI - - !Ref 'AWS::Region' - - 'linuxamd64' - BlockDeviceMappings: - - DeviceName: !If [ UseDefaultRootVolumeName, !If [ UseWindowsAgents, /dev/sda1, /dev/xvda ], !Ref RootVolumeName ] - Ebs: { VolumeSize: !Ref RootVolumeSize, VolumeType: !Ref RootVolumeType } - TagSpecifications: - - ResourceType: instance - Tags: - - Key: Role - Value: buildkite-agent - - Key: Name - Value: buildkite-agent - - Key: BuildkiteAgentRelease - Value: !Ref BuildkiteAgentRelease - - Key: BuildkiteQueue - Value: !Ref BuildkiteQueue - - !If - - UseCostAllocationTags - - Key: !Ref CostAllocationTagName - Value: !Ref CostAllocationTagValue - - !Ref "AWS::NoValue" - UserData: - Fn::Base64: !If - - UseWindowsAgents - - !Sub - - | - - $Env:DOCKER_USERNS_REMAP="${EnableDockerUserNamespaceRemap}" - $Env:DOCKER_EXPERIMENTAL="${EnableDockerExperimental}" - powershell -file C:\buildkite-agent\bin\bk-configure-docker.ps1 >> C:\buildkite-agent\elastic-stack.log - - $Env:BUILDKITE_STACK_NAME="${AWS::StackName}" - $Env:BUILDKITE_STACK_VERSION="v5.8.0" - $Env:BUILDKITE_SCALE_IN_IDLE_PERIOD="${ScaleInIdlePeriod}" - $Env:BUILDKITE_SECRETS_BUCKET="${LocalSecretsBucket}" - $Env:BUILDKITE_SECRETS_BUCKET_REGION="${LocalSecretsBucketRegion}" - $Env:BUILDKITE_AGENT_TOKEN_PATH="${AgentTokenPath}" - $Env:BUILDKITE_AGENTS_PER_INSTANCE="${AgentsPerInstance}" - $Env:BUILDKITE_AGENT_TAGS="${BuildkiteAgentTags}" - $Env:BUILDKITE_AGENT_TIMESTAMP_LINES="${BuildkiteAgentTimestampLines}" - $Env:BUILDKITE_AGENT_EXPERIMENTS="${BuildkiteAgentExperiments}" - $Env:BUILDKITE_AGENT_RELEASE="${BuildkiteAgentRelease}" - $Env:BUILDKITE_QUEUE="${BuildkiteQueue}" - $Env:BUILDKITE_AGENT_ENABLE_GIT_MIRRORS_EXPERIMENT="${EnableAgentGitMirrorsExperiment}" - $Env:BUILDKITE_ELASTIC_BOOTSTRAP_SCRIPT="${BootstrapScriptUrl}" - $Env:BUILDKITE_AUTHORIZED_USERS_URL="${AuthorizedUsersUrl}" - $Env:BUILDKITE_ECR_POLICY="${ECRAccessPolicy}" - $Env:BUILDKITE_TERMINATE_INSTANCE_AFTER_JOB="${BuildkiteTerminateInstanceAfterJob}" - $Env:BUILDKITE_ADDITIONAL_SUDO_PERMISSIONS="${BuildkiteAdditionalSudoPermissions}" - $Env:BUILDKITE_WINDOWS_ADMINISTRATOR="${BuildkiteWindowsAdministrator}" - $Env:AWS_DEFAULT_REGION="${AWS::Region}" - $Env:SECRETS_PLUGIN_ENABLED="${EnableSecretsPlugin}" - $Env:ECR_PLUGIN_ENABLED="${EnableECRPlugin}" - $Env:DOCKER_LOGIN_PLUGIN_ENABLED="${EnableDockerLoginPlugin}" - $Env:AWS_REGION="${AWS::Region}" - powershell -file C:\buildkite-agent\bin\bk-install-elastic-stack.ps1 >> C:\buildkite-agent\elastic-stack.log - - - { - LocalSecretsBucket: !If [ CreateSecretsBucket, !Ref ManagedSecretsBucket, !Ref SecretsBucket ], - LocalSecretsBucketRegion: !If [ CreateSecretsBucket, !Ref AWS::Region, !Ref SecretsBucketRegion ], - AgentTokenPath: !If [ UseCustomerManagedParameterPath, !Ref BuildkiteAgentTokenParameterStorePath, !Ref BuildkiteAgentTokenParameter ], - } - - !Sub - - | - Content-Type: multipart/mixed; boundary="==BOUNDARY==" - MIME-Version: 1.0 - --==BOUNDARY== - Content-Type: text/cloud-boothook; charset="us-ascii" - BUILDKITE_ENABLE_INSTANCE_STORAGE="${EnableInstanceStorage}" \ - /usr/local/bin/bk-mount-instance-storage.sh - --==BOUNDARY== - Content-Type: text/cloud-boothook; charset="us-ascii" - DOCKER_USERNS_REMAP=${EnableDockerUserNamespaceRemap} \ - DOCKER_EXPERIMENTAL=${EnableDockerExperimental} \ - BUILDKITE_ENABLE_INSTANCE_STORAGE="${EnableInstanceStorage}" \ - /usr/local/bin/bk-configure-docker.sh - --==BOUNDARY== - Content-Type: text/x-shellscript; charset="us-ascii" - #!/bin/bash -v - BUILDKITE_STACK_NAME="${AWS::StackName}" \ - BUILDKITE_STACK_VERSION="v5.8.0" \ - BUILDKITE_SCALE_IN_IDLE_PERIOD="${ScaleInIdlePeriod}" \ - BUILDKITE_SECRETS_BUCKET="${LocalSecretsBucket}" \ - BUILDKITE_SECRETS_BUCKET_REGION="${LocalSecretsBucketRegion}" \ - BUILDKITE_AGENT_TOKEN_PATH="${AgentTokenPath}" \ - BUILDKITE_AGENTS_PER_INSTANCE="${AgentsPerInstance}" \ - BUILDKITE_AGENT_TAGS="${BuildkiteAgentTags}" \ - BUILDKITE_AGENT_TIMESTAMP_LINES="${BuildkiteAgentTimestampLines}" \ - BUILDKITE_AGENT_EXPERIMENTS="${BuildkiteAgentExperiments}" \ - BUILDKITE_AGENT_RELEASE="${BuildkiteAgentRelease}" \ - BUILDKITE_QUEUE="${BuildkiteQueue}" \ - BUILDKITE_AGENT_ENABLE_GIT_MIRRORS_EXPERIMENT="${EnableAgentGitMirrorsExperiment}" \ - BUILDKITE_ELASTIC_BOOTSTRAP_SCRIPT="${BootstrapScriptUrl}" \ - BUILDKITE_ENABLE_INSTANCE_STORAGE="${EnableInstanceStorage}" \ - BUILDKITE_AUTHORIZED_USERS_URL="${AuthorizedUsersUrl}" \ - BUILDKITE_ECR_POLICY="${ECRAccessPolicy}" \ - BUILDKITE_TERMINATE_INSTANCE_AFTER_JOB="${BuildkiteTerminateInstanceAfterJob}" \ - BUILDKITE_ADDITIONAL_SUDO_PERMISSIONS="${BuildkiteAdditionalSudoPermissions}" \ - AWS_DEFAULT_REGION="${AWS::Region}" \ - SECRETS_PLUGIN_ENABLED="${EnableSecretsPlugin}" \ - ECR_PLUGIN_ENABLED="${EnableECRPlugin}" \ - DOCKER_LOGIN_PLUGIN_ENABLED="${EnableDockerLoginPlugin}" \ - DOCKER_EXPERIMENTAL="${EnableDockerExperimental}" \ - AWS_REGION="${AWS::Region}" \ - /usr/local/bin/bk-install-elastic-stack.sh - --==BOUNDARY==-- - - { - LocalSecretsBucket: !If [ CreateSecretsBucket, !Ref ManagedSecretsBucket, !Ref SecretsBucket ], - LocalSecretsBucketRegion: !If [ CreateSecretsBucket, !Ref AWS::Region, !Ref SecretsBucketRegion ], - AgentTokenPath: !If [ UseCustomerManagedParameterPath, !Ref BuildkiteAgentTokenParameterStorePath, !Ref BuildkiteAgentTokenParameter ], - } - - AgentAutoScaleGroup: - Type: AWS::AutoScaling::AutoScalingGroup - DependsOn: - - IAMPolicies - - VpcComplete - Properties: - VPCZoneIdentifier: !If [ "CreateVpcResources", [ !Ref Subnet0, !Ref Subnet1 ], !Ref Subnets ] - MixedInstancesPolicy: - InstancesDistribution: - OnDemandPercentageAboveBaseCapacity: !If [ SpotPriceSet, 0, !Ref OnDemandPercentage ] - SpotAllocationStrategy: capacity-optimized - SpotMaxPrice: !If [SpotPriceSet, !Ref SpotPrice, !Ref "AWS::NoValue"] - LaunchTemplate: - LaunchTemplateSpecification: - LaunchTemplateId: !Ref AgentLaunchTemplate - Version: !GetAtt "AgentLaunchTemplate.LatestVersionNumber" - Overrides: - - InstanceType: !Select [ "0", !Split [ ",", !Join [ ",", [ !Ref InstanceType, "", "", "" ] ] ] ] - - !If - - UseInstanceType2 - - InstanceType: !Select [ "1", !Split [ ",", !Join [ ",", [ !Ref InstanceType, "", "", "" ] ] ] ] - - !Ref "AWS::NoValue" - - !If - - UseInstanceType3 - - InstanceType: !Select [ "2", !Split [ ",", !Join [ ",", [ !Ref InstanceType, "", "", "" ] ] ] ] - - !Ref "AWS::NoValue" - - !If - - UseInstanceType4 - - InstanceType: !Select [ "3", !Split [ ",", !Join [ ",", [ !Ref InstanceType, "", "", "" ] ] ] ] - - !Ref "AWS::NoValue" - MinSize: !Ref MinSize - MaxSize: !Ref MaxSize - Cooldown: 60 - MetricsCollection: - - Granularity: 1Minute - Metrics: - - GroupMinSize - - GroupMaxSize - - GroupInServiceInstances - - GroupTerminatingInstances - - GroupPendingInstances - TerminationPolicies: - - OldestLaunchConfiguration - - ClosestToNextInstanceHour - NewInstancesProtectedFromScaleIn: true - CreationPolicy: - ResourceSignal: - Timeout: !If [ UseDefaultInstanceCreationTimeout, !If [ UseWindowsAgents, PT10M, PT5M ], !Ref InstanceCreationTimeout ] - Count: !Ref MinSize - UpdatePolicy: - AutoScalingReplacingUpdate: - WillReplace: true - - AsgProcessSuspenderRole: - Type: AWS::IAM::Role - Properties: - PermissionsBoundary: !If [ SetInstanceRolePermissionsBoundaryARN, !Ref InstanceRolePermissionsBoundaryARN, !Ref "AWS::NoValue" ] - AssumeRolePolicyDocument: - Version: 2012-10-17 - Statement: - - Effect: Allow - Principal: - Service: - - lambda.amazonaws.com - Action: - - sts:AssumeRole - ManagedPolicyArns: - - arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole - Policies: - - PolicyName: AsgProcessModification - PolicyDocument: - Version: 2012-10-17 - Statement: - - Effect: Allow - Action: - - 'autoscaling:SuspendProcesses' - Resource: !Sub arn:${AWS::Partition}:autoscaling:${AWS::Region}:${AWS::AccountId}:autoScalingGroup:*:autoScalingGroupName/${AWS::StackName}-AgentAutoScaleGroup-* - - AzRebalancingSuspenderFunction: - Type: AWS::Lambda::Function - Properties: - Description: 'Disables AZ Rebalancing on the agent ASG' - Code: - ZipFile: | - import cfnresponse - import boto3 - def handler(event, context): - try: - if event['RequestType'] == 'Delete': - cfnresponse.send(event, context, cfnresponse.SUCCESS, {}, "CustomResourcePhysicalID") - else: - client = boto3.client('autoscaling') - props = event['ResourceProperties'] - response = client.suspend_processes(AutoScalingGroupName=props['AutoScalingGroupName'], ScalingProcesses=['AZRebalance']) - cfnresponse.send(event, context, cfnresponse.SUCCESS, {}, "CustomResourcePhysicalID") - except BaseException as err: - print('ERROR: ', err) - cfnresponse.send(event, context, cfnresponse.FAILED, {}, "CustomResourcePhysicalID") - Handler: index.handler - Role: !GetAtt AsgProcessSuspenderRole.Arn - Runtime: 'python3.7' - - AzRebalancingSuspender: - Type: AWS::CloudFormation::CustomResource - Version: 1.0 - Properties: - ServiceToken: !GetAtt AzRebalancingSuspenderFunction.Arn - AutoScalingGroupName: !Ref AgentAutoScaleGroup - - SecurityGroup: - Type: AWS::EC2::SecurityGroup - Condition: CreateSecurityGroup - Properties: - GroupDescription: Enable access to agents - VpcId: !If [ "CreateVpcResources", !Ref Vpc, !Ref VpcId ] - Tags: - - Key: Name - Value: !Ref 'AWS::StackName' - - SecurityGroupSshIngress: - Condition: EnableSshIngress - Type: AWS::EC2::SecurityGroupIngress - Properties: - GroupId: !GetAtt SecurityGroup.GroupId - IpProtocol: tcp - FromPort: 22 - ToPort: 22 - CidrIp: 0.0.0.0/0 - - Autoscaling: - Type: AWS::Serverless::Application - Condition: HasVariableSize - Properties: - Location: - ApplicationId: arn:aws:serverlessrepo:us-east-1:172840064832:applications/buildkite-agent-scaler - SemanticVersion: '1.1.3' - Parameters: - BuildkiteAgentTokenParameter: !If [ UseCustomerManagedParameterPath, !Ref BuildkiteAgentTokenParameterStorePath, !Ref BuildkiteAgentTokenParameter ] - BuildkiteAgentTokenParameterStoreKMSKey: !If [ UseCustomerManagedKeyForParameterStore, !Ref BuildkiteAgentTokenParameterStoreKMSKey, "" ] - BuildkiteQueue: !Ref BuildkiteQueue - AgentsPerInstance: !Ref AgentsPerInstance - MinSize: !Ref MinSize - MaxSize: !Ref MaxSize - AgentAutoScaleGroup: !Ref AgentAutoScaleGroup - ScaleOutFactor: !Ref ScaleOutFactor - ScaleOutForWaitingJobs: !Ref ScaleOutForWaitingJobs diff --git a/terraform/buildkite.tf b/terraform/buildkite.tf index 7d36003..65b893f 100644 --- a/terraform/buildkite.tf +++ b/terraform/buildkite.tf @@ -1,53 +1,7 @@ -resource "aws_cloudformation_stack" "buildkite" { - name = "buildkite-elasticstack" - - capabilities = ["CAPABILITY_NAMED_IAM", "CAPABILITY_AUTO_EXPAND"] - - parameters = { - MinSize = 0 - MaxSize = 20 - - SpotPrice = 0.05 - InstanceType = "r5.large" - - BuildkiteQueue = "default" - - # This setting tells Buildkite that: - # - # - it should turn off an instance if it's idle for 10 minutes (=600s) - # - it should pre-emptively start instances for jobs that are behind - # a 'wait' step - # - # This is a new feature we got when we updated to v5.7.2 of the - # CloudFormation template (22 November 2021). I'm enabling it to see - # if it makes a difference in Scala repos where we do one autoformat step - # and then fan out to the main build. - # - ScaleOutForWaitingJobs = true - ScaleInIdlePeriod = 600 - - # We don't have to terminate an agent after a job completes. We have - # an agent hook (see buildkite_agent_hook.sh) which tries to clean up - # any state left over from previous jobs, so each instance will be "fresh", - # but already have a local cache of Docker images and Scala libraries. - BuildkiteTerminateInstanceAfterJob = false - - InstanceRoleName = local.ci_agent_role_name - - RootVolumeSize = 25 - RootVolumeName = "/dev/xvda" - RootVolumeType = "gp2" - - # If we don't disable this setting, we get this error when trying to - # run Docker containers on the instances: - # - # docker: Error response from daemon: cannot share the host's - # network namespace when user namespaces are enabled. - # - EnableDockerUserNamespaceRemap = false - - # This is a collection of settings that should be the same for every - # instance of the Buildkite stack. +locals { + # This is a collection of settings that should be the same for every + # instance of our Buildkite stack. + common_parameters = { AgentsPerInstance = 1 BuildkiteAgentTokenParameterStorePath = "/aws/reference/secretsmanager/builds/buildkite_agent_key" @@ -69,7 +23,60 @@ resource "aws_cloudformation_stack" "buildkite" { BuildkiteAgentRelease = "stable" BuildkiteAgentTimestampLines = false + + # We don't have to terminate an agent after a job completes. We have + # an agent hook (see buildkite_agent_hook.sh) which tries to clean up + # any state left over from previous jobs, so each instance will be "fresh", + # but already have a local cache of Docker images and Scala libraries. + BuildkiteTerminateInstanceAfterJob = false } +} + +resource "aws_cloudformation_stack" "buildkite" { + name = "buildkite-elasticstack" + + capabilities = ["CAPABILITY_NAMED_IAM", "CAPABILITY_AUTO_EXPAND"] + + parameters = merge( + { + MinSize = 0 + MaxSize = 20 + + SpotPrice = 0.05 + InstanceType = "r5.large" + + BuildkiteQueue = "default" + + # This setting tells Buildkite that: + # + # - it should turn off an instance if it's idle for 10 minutes (=600s) + # - it should pre-emptively start instances for jobs that are behind + # a 'wait' step + # + # This is a new feature we got when we updated to v5.7.2 of the + # CloudFormation template (22 November 2021). I'm enabling it to see + # if it makes a difference in Scala repos where we do one autoformat step + # and then fan out to the main build. + # + ScaleOutForWaitingJobs = true + ScaleInIdlePeriod = 600 + + InstanceRoleName = local.ci_agent_role_name + + RootVolumeSize = 25 + RootVolumeName = "/dev/xvda" + RootVolumeType = "gp2" + + # If we don't disable this setting, we get this error when trying to + # run Docker containers on the instances: + # + # docker: Error response from daemon: cannot share the host's + # network namespace when user namespaces are enabled. + # + EnableDockerUserNamespaceRemap = false + }, + local.common_parameters + ) template_body = file("${path.module}/buildkite-v5.16.1.yml") } @@ -89,69 +96,42 @@ resource "aws_cloudformation_stack" "buildkite_scala" { capabilities = ["CAPABILITY_NAMED_IAM", "CAPABILITY_AUTO_EXPAND"] - parameters = { - SpotPrice = 0.2 - InstanceType = "c5.2xlarge" - - BuildkiteQueue = "scala" - - MinSize = 0 - MaxSize = 60 - - # This setting would tell Buildkite to scale out for steps behind wait - # steps. - # - # We don't enable it for nano instances because these are often waiting - # behind long-running tasks in the large queue (e.g. build and publish - # a Docker image, then deploy it from a nano instance) and the pre-emptively - # scaled instances would likely time out before they were used. - # - ScaleOutForWaitingJobs = false - - # We don't have to terminate an agent after a job completes. We have - # an agent hook (see buildkite_agent_hook.sh) which tries to clean up - # any state left over from previous jobs, so each instance will be "fresh", - # but already have a local cache of Docker images and Scala libraries. - BuildkiteTerminateInstanceAfterJob = false - - # If we don't disable this setting, we get this error when trying to - # run Docker containers on the instances: - # - # docker: Error response from daemon: cannot share the host's - # network namespace when user namespaces are enabled. - # - EnableDockerUserNamespaceRemap = false - - InstanceRoleName = local.ci_scala_agent_role_name - - RootVolumeSize = 25 - RootVolumeName = "/dev/xvda" - RootVolumeType = "gp2" - - # This is a collection of settings that should be the same for every - # instance of the Buildkite stack. - AgentsPerInstance = 1 - - BuildkiteAgentTokenParameterStorePath = "/aws/reference/secretsmanager/builds/buildkite_agent_key" - - InstanceCreationTimeout = "PT5M" - - VpcId = local.ci_vpc_id - Subnets = join(",", local.ci_vpc_private_subnets) - SecurityGroupId = aws_security_group.buildkite.id - - CostAllocationTagName = "aws:createdBy" - CostAllocationTagValue = "buildkite-elasticstack" - - # This tells Buildkite to fetch secrets from our S3 bucket, which - # includes the agent hook and SSH key. - EnableSecretsPlugin = true - - SecretsBucket = aws_s3_bucket.buildkite_secrets.id - - BuildkiteAgentRelease = "stable" - BuildkiteAgentTimestampLines = false - } + parameters = merge( + { + SpotPrice = 0.2 + InstanceType = "c5.2xlarge" + + BuildkiteQueue = "scala" + + MinSize = 0 + MaxSize = 60 + + # This setting would tell Buildkite to scale out for steps behind wait + # steps. + # + # We don't enable it for nano instances because these are often waiting + # behind long-running tasks in the large queue (e.g. build and publish + # a Docker image, then deploy it from a nano instance) and the pre-emptively + # scaled instances would likely time out before they were used. + # + ScaleOutForWaitingJobs = false + + # If we don't disable this setting, we get this error when trying to + # run Docker containers on the instances: + # + # docker: Error response from daemon: cannot share the host's + # network namespace when user namespaces are enabled. + # + EnableDockerUserNamespaceRemap = false + + InstanceRoleName = local.ci_scala_agent_role_name + + RootVolumeSize = 25 + RootVolumeName = "/dev/xvda" + RootVolumeType = "gp2" + }, + local.common_parameters + ) // The catalogue pipeline batcher only seems to work with this version. // See: https://github.com/wellcomecollection/platform/issues/5656 @@ -177,71 +157,44 @@ resource "aws_cloudformation_stack" "buildkite_nano" { capabilities = ["CAPABILITY_NAMED_IAM", "CAPABILITY_AUTO_EXPAND"] - parameters = { - SpotPrice = 0.01 - InstanceType = "t3.nano" - - BuildkiteQueue = "nano" - - # At time of writing (1 October 2021), we have six deployment tasks - # in the pipeline repo: four adapters, the reindexer, and the pipeline. - # - # We want all of these to run simultaneously and leave room for other - # nano tasks, so we need >6 instances. - # - # We always run at least one nano instance because nano instances are - # extremely cheap, and this means the initial "pipeline upload" step - # is always warm. An on-demand t3.nano costs ~$4 a month, and we use - # spot pricing, so this is unlikely to be an issue. - MinSize = 1 - MaxSize = 10 - - # This setting would tell Buildkite to scale out for steps behind wait - # steps. - # - # We don't enable it for nano instances because these are often waiting - # behind long-running tasks in the large queue (e.g. build and publish - # a Docker image, then deploy it from a nano instance) and the pre-emptively - # scaled instances would likely time out before they were used. - # - ScaleOutForWaitingJobs = false - - # We don't have to terminate an agent after a job completes. We have - # an agent hook (see buildkite_agent_hook.sh) which tries to clean up - # any state left over from previous jobs, so each instance will be "fresh", - # but already have a local cache of Docker images and Scala libraries. - BuildkiteTerminateInstanceAfterJob = false - - InstanceRoleName = local.ci_nano_agent_role_name - - RootVolumeSize = 10 - RootVolumeName = "/dev/xvda" - RootVolumeType = "gp2" - - # This is a collection of settings that should be the same for every - # instance of the Buildkite stack. - AgentsPerInstance = 1 - - BuildkiteAgentTokenParameterStorePath = "/aws/reference/secretsmanager/builds/buildkite_agent_key" - - InstanceCreationTimeout = "PT5M" - - VpcId = local.ci_vpc_id - Subnets = join(",", local.ci_vpc_private_subnets) - SecurityGroupId = aws_security_group.buildkite.id - - CostAllocationTagName = "aws:createdBy" - CostAllocationTagValue = "buildkite-elasticstack" - - # This tells Buildkite to fetch secrets from our S3 bucket, which - # includes the agent hook and SSH key. - EnableSecretsPlugin = true - - SecretsBucket = aws_s3_bucket.buildkite_secrets.id - - BuildkiteAgentRelease = "stable" - BuildkiteAgentTimestampLines = false - } + parameters = merge( + { + SpotPrice = 0.01 + InstanceType = "t3.nano" + + BuildkiteQueue = "nano" + + # At time of writing (1 October 2021), we have six deployment tasks + # in the pipeline repo: four adapters, the reindexer, and the pipeline. + # + # We want all of these to run simultaneously and leave room for other + # nano tasks, so we need >6 instances. + # + # We always run at least one nano instance because nano instances are + # extremely cheap, and this means the initial "pipeline upload" step + # is always warm. An on-demand t3.nano costs ~$4 a month, and we use + # spot pricing, so this is unlikely to be an issue. + MinSize = 1 + MaxSize = 10 + + # This setting would tell Buildkite to scale out for steps behind wait + # steps. + # + # We don't enable it for nano instances because these are often waiting + # behind long-running tasks in the large queue (e.g. build and publish + # a Docker image, then deploy it from a nano instance) and the pre-emptively + # scaled instances would likely time out before they were used. + # + ScaleOutForWaitingJobs = false + + InstanceRoleName = local.ci_nano_agent_role_name + + RootVolumeSize = 10 + RootVolumeName = "/dev/xvda" + RootVolumeType = "gp2" + }, + local.common_parameters + ) template_body = file("${path.module}/buildkite-v5.16.1.yml") } diff --git a/terraform/buildkite_agent_hooks.tf b/terraform/buildkite_agent_hooks.tf index c1465cc..d956438 100644 --- a/terraform/buildkite_agent_hooks.tf +++ b/terraform/buildkite_agent_hooks.tf @@ -11,7 +11,7 @@ locals { buildkite_agent_hook_path = "${path.module}/../buildkite_agent_hook.sh" } -resource "aws_s3_bucket_object" "agent_hook" { +resource "aws_s3_object" "agent_hook" { bucket = aws_s3_bucket.buildkite_secrets.id key = "env" source = local.buildkite_agent_hook_path diff --git a/terraform/provider.tf b/terraform/provider.tf index f09aeab..5959f40 100644 --- a/terraform/provider.tf +++ b/terraform/provider.tf @@ -21,7 +21,7 @@ provider "aws" { default_tags { tags = { - TerraformConfigurationURL = "https://github.com/wellcomecollection/platform-infrastructure/tree/main/builds" + TerraformConfigurationURL = "https://github.com/wellcomecollection/buildkite-infrastructure/" Environment = "Production" Department = "Digital Platform" Division = "Wellcome Collection" diff --git a/terraform/s3.tf b/terraform/s3.tf index 304ac1e..b48e4b7 100644 --- a/terraform/s3.tf +++ b/terraform/s3.tf @@ -4,15 +4,25 @@ locals { resource "aws_s3_bucket" "buildkite_secrets" { bucket = "wellcomecollection-buildkite-secrets" +} + +resource "aws_s3_bucket_acl" "buildkite_secrets" { + bucket = aws_s3_bucket.buildkite_secrets.id acl = "private" +} - logging { - target_bucket = aws_s3_bucket.buildkite_secrets_logging.id - } +resource "aws_s3_bucket_logging" "buildkite_secrets" { + bucket = aws_s3_bucket.buildkite_secrets.id + target_bucket = aws_s3_bucket.buildkite_secrets_logging.id + target_prefix = "" } resource "aws_s3_bucket" "buildkite_secrets_logging" { bucket = "wellcomecollection-buildkite-secrets-logging" +} + +resource "aws_s3_bucket_acl" "buildkite_secrets_logging" { + bucket = aws_s3_bucket.buildkite_secrets_logging.id acl = "log-delivery-write" }