From 1cee2041f19f76efb574896d233a9f6d5ff82b6b Mon Sep 17 00:00:00 2001 From: pgodithi Date: Thu, 5 May 2022 12:55:32 -0400 Subject: [PATCH 01/13] parent 83a6200a1a322c7fa5b2ba9ea3a2ce80333b50dd author pgodithi 1651769732 -0400 committer pgodithi 1658274990 -0400 gpgsig -----BEGIN PGP SIGNATURE----- iQIzBAABCAAdFiEEuaPMeSek3p9IWNzDMSDf0RO9/aIFAmLXRK4ACgkQMSDf0RO9 /aKKpQ/9FQ92spUtBEdt2lf4Kz6c2x7+eunr5kzzxZPB8fj28AFWBtGP/NA6a/WK BjRgtxZ4060PCDUtbcnfFy8mg8PdQa03FpofQKr6exSfDTI+TNISJAkcI2HIWzqx ehiFRDGWG577fgrnk67xxd6LxD8sO8EHWBJzZWKrYA3z7f0Q05n1q2osm/jhDYgK cEDQBHZJVi4htO2sBGJO3caqCbqt0EphxYy+uua5On4a2ZgWlBUxjqoaZ6X/LrnR 6IoN8PYVUtyrkbWkkXB+0T4PPDYP8DwnCrCsDQxQyYQ7tBqnR3hQ6nfga2d4Dry5 Dz9oqPoWZjDJNBAf+B6NAmwmyIZHn2lnVL/b8GvwGjSOixyUqaptRKVrg/PwIT+I i2CeMuGIxIG8pycC+wruSNoNqV26Vm7rFSOC4VSvOXneQUStMrAEM+LjcCDtG8bF 0837RaB4Vj37QfkXJyQl2JLTvw6TUKbZuCXqSTLF6oy4nfKN1wBPhfmgjRt146o3 VKt9iZ/q5kOEAPhEHJkHWoC/C+jaMRkVk0ZV5CoujQcIPZX2HfA88RogNZe0Ommy ayc+gLw8oBiRBeKkWuQLrLtrBYCLKrIK5qjEEj9MqLxbLSlDRXJjvZvPSUGqS2wg a8KKD0i9faPpRIuViJRqHztQBXVsv0UsYFMeeid00lJm2F4dmBk= =c8LN -----END PGP SIGNATURE----- fixed IAM roles and code cleanup Signed-off-by: pgodithi Fix tests Signed-off-by: pgodithi Added assume role feature Signed-off-by: pgodithi Added assume role feature Signed-off-by: pgodithi Added assume role feature Signed-off-by: pgodithi Added assume role feature Signed-off-by: pgodithi Added assume role feature Signed-off-by: pgodithi Added 2 new plugins Signed-off-by: pgodithi Add readme doc Signed-off-by: pgodithi Added docker resources Signed-off-by: pgodithi Added docker resources Signed-off-by: pgodithi Bump minor version Signed-off-by: pgodithi change docker tag to 2.332.3-lts Signed-off-by: pgodithi added jenkinsOpts Signed-off-by: pgodithi added jenkinsOpts Signed-off-by: pgodithi added jenkinsOpts Signed-off-by: pgodithi Added IAM required imports Signed-off-by: pgodithi Added agent node unit tests Signed-off-by: pgodithi Added agent node unit tests Signed-off-by: pgodithi Added Managed SSM Policy to Agent instance role (#117) Signed-off-by: Rishabh Singh fix agentAssumeRole Signed-off-by: pgodithi Ecr public Signed-off-by: pgodithi Added AmazonSSMManagedInstanceCore Signed-off-by: pgodithi Added AmazonSSMManagedInstanceCore policy to agents. (#124) * fixed IAM roles and code cleanup Signed-off-by: pgodithi * Fix tests Signed-off-by: pgodithi * Added assume role feature Signed-off-by: pgodithi * Added assume role feature Signed-off-by: pgodithi * Added assume role feature Signed-off-by: pgodithi * Added assume role feature Signed-off-by: pgodithi * Added assume role feature Signed-off-by: pgodithi * Added 2 new plugins Signed-off-by: pgodithi * Add readme doc Signed-off-by: pgodithi * Added docker resources Signed-off-by: pgodithi * Added docker resources Signed-off-by: pgodithi * Bump minor version Signed-off-by: pgodithi * change docker tag to 2.332.3-lts Signed-off-by: pgodithi * added jenkinsOpts Signed-off-by: pgodithi * added jenkinsOpts Signed-off-by: pgodithi * added jenkinsOpts Signed-off-by: pgodithi * Added IAM required imports Signed-off-by: pgodithi * Added agent node unit tests Signed-off-by: pgodithi * Added agent node unit tests Signed-off-by: pgodithi * fix agentAssumeRole Signed-off-by: pgodithi * Ecr public Signed-off-by: pgodithi * Added AmazonSSMManagedInstanceCore Signed-off-by: pgodithi Added new jenkins agents and logging bug-fix (#126) Signed-off-by: Rishabh Singh Added new output parameters and updated jenkins image tag (#128) Signed-off-by: Rishabh Singh use jenkins-2.332.3 jdk8 base image (#129) Signed-off-by: Rishabh Singh added support for QEMU emulators (#131) Signed-off-by: Rishabh Singh Add Docker restart policy (#137) * fixed IAM roles and code cleanup Signed-off-by: pgodithi * Fix tests Signed-off-by: pgodithi * Added assume role feature Signed-off-by: pgodithi * Added assume role feature Signed-off-by: pgodithi * Added assume role feature Signed-off-by: pgodithi * Added assume role feature Signed-off-by: pgodithi * Added assume role feature Signed-off-by: pgodithi * Added 2 new plugins Signed-off-by: pgodithi * Add readme doc Signed-off-by: pgodithi * Added docker resources Signed-off-by: pgodithi * Added docker resources Signed-off-by: pgodithi * Bump minor version Signed-off-by: pgodithi * change docker tag to 2.332.3-lts Signed-off-by: pgodithi * added jenkinsOpts Signed-off-by: pgodithi * added jenkinsOpts Signed-off-by: pgodithi * added jenkinsOpts Signed-off-by: pgodithi * Added IAM required imports Signed-off-by: pgodithi * Added agent node unit tests Signed-off-by: pgodithi * Added agent node unit tests Signed-off-by: pgodithi * fix agentAssumeRole Signed-off-by: pgodithi * Ecr public Signed-off-by: pgodithi * Added AmazonSSMManagedInstanceCore Signed-off-by: pgodithi * added docker-compose restart policy Signed-off-by: pgodithi Add markup formatter as safe/html disable syntax highlighting to allow html (#138) Signed-off-by: Peter Zhu Set cache to false for aws secret manager plugin (#140) Signed-off-by: Peter Zhu Add environment variables using configuration as code (#136) * Add environment variables using configuration as code Signed-off-by: Sayali Gaikawad Install workflow-api plugin to get latest version and remove lockable permissions (#141) * Install workflow-api plugin to get latest version Signed-off-by: Sayali Gaikawad * Remove lockable permissions Signed-off-by: Sayali Gaikawad [Bug-fix]: Handle multiple colons and space (#142) * Handle multiple colons and space Signed-off-by: Sayali Gaikawad * Handle multiple space Signed-off-by: Sayali Gaikawad Move env variabled to yaml (#143) * Move to yaml Signed-off-by: Sayali Gaikawad add ubuntu agent and cdn with lambda@edge resource for public access (#134) Signed-off-by: Rishabh Singh Reload configuration as code via curl instead of cli (#145) * Reload configuration as code via curl instead of cli Signed-off-by: Sayali Gaikawad Main/Agent Node new setups on tag and executors (#147) * Main/Agent Node new setups on tag and executors Signed-off-by: Peter Zhu * Remove DCO check as it is replaced by dco app Signed-off-by: Peter Zhu * Remove stack in node ts as it is not used anymore Signed-off-by: Peter Zhu Fix Ubuntu agent init script bug (#148) Signed-off-by: Peter Zhu Add Ubuntu Single Host for Gradle Check (#149) * Add Ubuntu Single Host for Gradle Check Signed-off-by: Peter Zhu * Change default idle time to 60min Signed-off-by: Peter Zhu Raise the gradle check runner to have c518xlarge as in Fork Jenkins (#150) Signed-off-by: Peter Zhu New cmds reduce AL2 failure and increase gradle check to c524xlarge runner (#151) Signed-off-by: Peter Zhu Add jdk14 as it is required by OS 1.x gradle check (#152) Signed-off-by: Peter Zhu Migrate perf test agent setups to public jenkins (#153) Signed-off-by: Peter Zhu Add Mac agent support Signed-off-by: pgodithi Add Mac agent support: Added README Signed-off-by: pgodithi Add Mac agent support: Added README Signed-off-by: pgodithi Add Mac agent support: Added README Signed-off-by: pgodithi Add Mac agent support: Added README Signed-off-by: pgodithi Add Mac agent support: Added README Signed-off-by: pgodithi Add Mac agent support: Added max-len, eqeqeq Signed-off-by: pgodithi Add Mac agent support: Added tests Signed-off-by: pgodithi Add Mac agent support: Added tests Signed-off-by: pgodithi Add Mac agent support: Added tests Signed-off-by: pgodithi --- .github/workflows/cdk.yml | 3 + .github/workflows/dco.yml | 18 - .github/workflows/docker.yml | 7 +- README.md | 61 ++- bin/ci-stack.ts | 21 +- docker/Dockerfile | 5 +- docker/plugins.txt | 6 +- lib/buildArtifacts/artifacts-public-access.ts | 69 +++ .../build-artifacts-permissions.ts | 91 ++++ lib/ci-cdn-stack.ts | 30 ++ lib/ci-config-stack.ts | 10 + lib/ci-ecr-stack.ts | 73 --- lib/ci-stack.ts | 31 +- lib/compute/agent-node-config.ts | 330 +++++++++---- lib/compute/agent-nodes.ts | 87 +++- lib/compute/env-config.ts | 39 ++ lib/compute/jenkins-main-node.ts | 82 ++-- lib/compute/oidc-config.ts | 7 +- lib/deploy-aws-assets.ts | 29 -- lib/network/ci-external-load-balancer.ts | 2 +- package-lock.json | 433 ++++++++++++++++-- package.json | 6 +- resources/baseJenkins.yaml | 42 +- resources/cf-url-rewriter/cf-url-rewriter.ts | 68 +++ resources/cf-url-rewriter/https-get.ts | 43 ++ resources/cf-url-rewriter/package-lock.json | 26 ++ resources/cf-url-rewriter/package.json | 17 + resources/docker-compose.yml | 17 +- test/ci-cdn-stack.test.ts | 59 +++ test/ci-ecr-stack.test.ts | 168 ------- test/ci-stack.test.ts | 15 +- test/compute/agent-node-config.test.ts | 148 ++++++ test/compute/env-config.test.ts | 39 ++ test/compute/jenkins-main-node.test.ts | 5 +- test/data/env.yaml | 4 + test/data/jenkins.yaml | 46 ++ test/data/test_env.yaml | 206 +++++++++ test/deploy-aws-assets.test.ts | 60 --- 38 files changed, 1793 insertions(+), 610 deletions(-) delete mode 100644 .github/workflows/dco.yml create mode 100644 lib/buildArtifacts/artifacts-public-access.ts create mode 100644 lib/buildArtifacts/build-artifacts-permissions.ts create mode 100644 lib/ci-cdn-stack.ts delete mode 100644 lib/ci-ecr-stack.ts create mode 100644 lib/compute/env-config.ts delete mode 100644 lib/deploy-aws-assets.ts create mode 100644 resources/cf-url-rewriter/cf-url-rewriter.ts create mode 100644 resources/cf-url-rewriter/https-get.ts create mode 100644 resources/cf-url-rewriter/package-lock.json create mode 100644 resources/cf-url-rewriter/package.json create mode 100644 test/ci-cdn-stack.test.ts delete mode 100644 test/ci-ecr-stack.test.ts create mode 100644 test/compute/agent-node-config.test.ts create mode 100644 test/compute/env-config.test.ts create mode 100644 test/data/env.yaml create mode 100644 test/data/test_env.yaml delete mode 100644 test/deploy-aws-assets.test.ts diff --git a/.github/workflows/cdk.yml b/.github/workflows/cdk.yml index 147da703..2ff646d1 100644 --- a/.github/workflows/cdk.yml +++ b/.github/workflows/cdk.yml @@ -15,4 +15,7 @@ jobs: - name: Run CDK Build and Test run: | npm install + cd resources/cf-url-rewriter + npm install + cd - npm run build diff --git a/.github/workflows/dco.yml b/.github/workflows/dco.yml deleted file mode 100644 index a8ed8651..00000000 --- a/.github/workflows/dco.yml +++ /dev/null @@ -1,18 +0,0 @@ -name: Developer Certificate of Origin Check - -on: [pull_request] - -jobs: - build: - runs-on: ubuntu-latest - - steps: - - name: Get PR Commits - id: 'get-pr-commits' - uses: tim-actions/get-pr-commits@v1.1.0 - with: - token: ${{ secrets.GITHUB_TOKEN }} - - name: DCO Check - uses: tim-actions/dco@v1.1.0 - with: - commits: ${{ steps.get-pr-commits.outputs.commits }} \ No newline at end of file diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 8852ea9b..5f286d76 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -2,7 +2,7 @@ name: Push new jenkins docker image on: push: - paths: + paths: - docker/** workflow_dispatch: @@ -10,6 +10,9 @@ jobs: docker: runs-on: ubuntu-latest steps: + - + name: Set up QEMU + uses: docker/setup-qemu-action@v2 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v1 @@ -26,4 +29,4 @@ jobs: platforms: linux/amd64,linux/arm64 context: "{{defaultContext}}:docker" push: true - tags: opensearchstaging/jenkins:2.332.2-lts,opensearchstaging/jenkins:latest \ No newline at end of file + tags: opensearchstaging/jenkins:2.332.3-lts-jdk8,opensearchstaging/jenkins:latest diff --git a/README.md b/README.md index 2db7b1c0..b9018725 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,10 @@ - [Executing Optional Tasks](#executing-optional-tasks) - [SSL Configuration](#ssl-configuration) - [Setup OpenId Connect (OIDC) via Federate](#setup-openid-connect-oidc-via-federate) - - [Data Retention](#Data Retention) + - [Data Retention](#data-retention) + - [Add environment variable](#add-environment-variables) + - [Assume role](#cross-account-assume-role) + - [Mac agents](#mac-agents) - [Troubleshooting](#troubleshooting) - [Main Node](#main-node) - [Useful commands](#useful-commands) @@ -41,12 +44,8 @@ OpenSearch Continuous Integration is an open source CI system for OpenSearch and new CIStack(app, 'CI-Beta', ciSettings, {}); ``` 3. Update the `ciSettings` according to the environment needs such as SSL or strict deployment, see [CIStackProps](./lib/ci-stack.ts) for details. -4. Import `DeployAwsAssets` stack to deploy aws assets as per your needs. -5. We currently support deploying public ECR and is deployed as follows - - ```typescript - new DeployAwsAssets(app, `OpenSearch-CI-Deploy-Assets`, {props); ``` -6. Update the `assetsSettings` according to the environment needs such as SSL or strict deployment, see [deployAwsAssetProps](./lib/ci-stack.ts) for details. -7. Deploy using the CI system of your choice. +4. Update the `assetsSettings` according to the environment needs such as SSL or strict deployment, see [deployAwsAssetProps](./lib/ci-stack.ts) for details. +5. Deploy using the CI system of your choice. ### Dev Deployment 1. Setup your local machine to credentials to deploy to the AWS Account @@ -58,6 +57,12 @@ OpenSearch Continuous Integration is an open source CI system for OpenSearch and `npm run cdk deploy OpenSearch-CI-Config-Dev -- -c useSsl=false -c runWithOidc=false` +1. Locate the secret manager arns in the ci-config-stack outputs for `CASC_RELOAD_TOKEN` and update the secret value ([see docs](https://docs.aws.amazon.com/cli/latest/reference/secretsmanager/put-secret-value.html)) with the password you want to use to reload jenkins configuration. _Do not enclose it in quotes_ +``` +$aws secretsmanager put-secret-value \ +--secret-id MyCASCreloadTokenSecretARN \ +--secret-string CascReloadToken +``` 1. [Optional](#ssl-configuration) Configure the elements of the config stack for SSL configuration 1. [Optional](#setup-openid-connect-oidc-via-federate) Configure the elements setting up oidc via federate 1. Deploy the ci-stack, takes ~10 minutes to deploy (parameter values depend on step 2 and step 3) @@ -68,17 +73,6 @@ OpenSearch Continuous Integration is an open source CI system for OpenSearch and 1. Go to the `OpenSearch-CI-Dev.JenkinsExternalLoadBalancerDns` url returned by CDK output to access the jenkins host. 1. If you want to destroy the stack make sure you delete the agent nodes manually (via jenkins UI or AWS console) so that shared resources (like vpc, security groups, etc) can be deleted. -### Deploying AWS assets -1. Setup your local machine to credentials to deploy to the AWS Account -2. Deploy the bootstrap stack by running following command that sets up required resources to create the stacks. [More info](https://docs.aws.amazon.com/cdk/latest/guide/bootstrapping.html) - - `npm run cdk bootstrap -- -c useSsl=false -c runWithOidc=false` -3. Deploy `deployAwsAssets` stack using the following (takes ~1 minute to deploy) - - 1. To deploy with ECR (default `false`) - - `npm run cdk deploy OpenSearch-CI-Deploy-Assets-Dev -- -c deployEcr=true` - - ### Executing Optional Tasks #### SSL Configuration 1. Locate the secret manager arns in the ci-config-stack outputs @@ -128,7 +122,36 @@ Change in any EC2 config (specially init config) leads to replacement of EC2. Th See inital [jenkins.yaml](./resources/baseJenkins.yaml) If you want to retain all the jobs and its build history, 1. Update the `dataRetention` property in `ciSettings` to true (defaults to false) see [CIStackProps](./lib/ci-stack.ts) for details. -This will create an EFS (Elastic File System) and mount it on `/var/lib/jenkins/jobs` which will retain all jobs and its build history. +This will create an EFS (Elastic File System) and mount it on `/var/lib/jenkins` which will retain all jobs and its build history. + +#### Add environment variables +Users can add global level environment variables using configuration as code as follows: + +Update the `envVarsFilePath` property in `ciSettings` to the *yaml* file path containing all environment variables in the form of key:value pair. See [CIStackProps](./lib/ci-stack.ts) for details. + +Example: See [env.txt](./test/data/env.yaml) +``` +envVarsFilePath = 'test/data/env.yaml' +``` + +#### Assume role +The Created jenkins agent role can assume cross account role by passing `agentAssumeRole` parameter +Example: +``` +npm run cdk deploy OpenSearch-CI-Dev -- -c useSsl=false -c runWithOidc=false -c agentAssumeRole=arn:aws:iam::522XXX13897:role/sample-assume-role +``` +NOTE: The assume role has to be pre-created for the agents to assume. Once CDK stack is deployed with `-c agentAssumeRole` flag, make sure this flag is passed for next CDK operations to make sure this created policy that assumes cross-account role is not removed. + +#### Mac agents +##### Prerequisite +To deploy mac agents, as a prerequisites make sure the backend AWS account has dedicated hosts setup done with instance family as `mac1` and instance type as `mac1.metal`. For More details check the [getting-started](https://aws.amazon.com/getting-started/hands-on/launch-connect-to-amazon-ec2-mac-instance/) guide. + +##### Configuration +To configure ec2 Mac agent setup run the stack with `-c macAgent=true`. +Example: +``` +npm run cdk deploy OpenSearch-CI-Dev -- -c useSsl=false -c runWithOidc=false -c macAgent=true +``` #### Runnning additional commands In cases where you need to run additional logic/commands, such as adding a cron to emit ssl cert expiry metric, you can pass the commands as a script using `additionalCommands` context parameter. diff --git a/bin/ci-stack.ts b/bin/ci-stack.ts index 0cd8a9a8..a961b482 100644 --- a/bin/ci-stack.ts +++ b/bin/ci-stack.ts @@ -6,10 +6,10 @@ * compatible open source license. */ -import { App, RemovalPolicy } from '@aws-cdk/core'; +import { App } from '@aws-cdk/core'; import { CIStack } from '../lib/ci-stack'; import { CIConfigStack } from '../lib/ci-config-stack'; -import { DeployAwsAssets } from '../lib/deploy-aws-assets'; +import { CiCdnStack } from '../lib/ci-cdn-stack'; const app = new App(); @@ -19,18 +19,5 @@ const ciConfigStack = new CIConfigStack(app, `OpenSearch-CI-Config-${defaultEnv} const ciStack = new CIStack(app, `OpenSearch-CI-${defaultEnv}`, {}); -new DeployAwsAssets(app, `OpenSearch-CI-Deploy-Assets-${defaultEnv}`, { - /* This will delete the ECR repository once the stack is destroyed. - * Default removal policy (if not specified) is RemovalPolicy.DESTROY */ - removalPolicy: RemovalPolicy.DESTROY, - mainNodeAccountNumber: ciStack.account, - envName: defaultEnv, - env: { - // public ECR repositories can only be created in us-east-1 - // https://github.com/aws/aws-cli/issues/5917#issuecomment-775564831 - region: 'us-east-1', - }, - repositories: [ - 'opensearch', - ], -}); +const ciCdnStack = new CiCdnStack(app, `OpenSearch-CI-Cdn-${defaultEnv}`, {}); +ciCdnStack.addDependency(ciStack); diff --git a/docker/Dockerfile b/docker/Dockerfile index 5d417e7a..5f9531c3 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,4 +1,5 @@ -FROM jenkins/jenkins:2.332.2-lts +FROM jenkins/jenkins:2.332.3-lts-jdk8 +LABEL maintainer="OpenSearch" ENV JAVA_OPTS -Djenkins.install.runSetupWizard=false COPY plugins.txt plugins.txt -RUN /usr/local/bin/install-plugins.sh < plugins.txt \ No newline at end of file +RUN /usr/local/bin/install-plugins.sh < plugins.txt diff --git a/docker/plugins.txt b/docker/plugins.txt index f4e78e20..af059986 100644 --- a/docker/plugins.txt +++ b/docker/plugins.txt @@ -61,7 +61,11 @@ throttle-concurrents timestamper uno-choice validating-string-parameter +workflow-api workflow-aggregator workflow-basic-steps workflow-step-api -ws-cleanup \ No newline at end of file +ws-cleanup +job-import-plugin +workflow-multibranch +pipeline-utility-steps diff --git a/lib/buildArtifacts/artifacts-public-access.ts b/lib/buildArtifacts/artifacts-public-access.ts new file mode 100644 index 00000000..d292c4aa --- /dev/null +++ b/lib/buildArtifacts/artifacts-public-access.ts @@ -0,0 +1,69 @@ +/** + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +import { Bucket } from '@aws-cdk/aws-s3'; +import { + CloudFrontAllowedMethods, CloudFrontWebDistribution, LambdaEdgeEventType, OriginAccessIdentity, +} from '@aws-cdk/aws-cloudfront'; +import { CanonicalUserPrincipal, PolicyStatement } from '@aws-cdk/aws-iam'; +import { NodejsFunction } from '@aws-cdk/aws-lambda-nodejs'; +import { Architecture, Runtime } from '@aws-cdk/aws-lambda'; +import { CfnOutput, Duration } from '@aws-cdk/core'; +import { CiCdnStack } from '../ci-cdn-stack'; + +export class ArtifactsPublicAccess { + constructor(stack: CiCdnStack, buildBucketArn: string) { + const buildBucket = Bucket.fromBucketArn(stack, 'artifactBuildBucket', `${buildBucketArn.toString()}`); + + const originAccessIdentity = new OriginAccessIdentity(stack, 'cloudfront-OAI', { + comment: `OAI for ${buildBucket.bucketName}`, + }); + + buildBucket.addToResourcePolicy(new PolicyStatement({ + actions: ['s3:GetObject'], + resources: [buildBucket.arnForObjects('*')], + principals: [new CanonicalUserPrincipal(originAccessIdentity.cloudFrontOriginAccessIdentityS3CanonicalUserId)], + })); + + const urlRewriter = new NodejsFunction(stack, 'CfUrlRewriter', { + runtime: Runtime.NODEJS_14_X, + entry: `${__dirname}/../../resources/cf-url-rewriter/cf-url-rewriter.ts`, + handler: 'handler', + memorySize: 128, + architecture: Architecture.X86_64, + }); + + const distro = new CloudFrontWebDistribution(stack, 'CloudFrontBuildBucket', { + originConfigs: [ + { + s3OriginSource: { + s3BucketSource: buildBucket, + originAccessIdentity, + }, + behaviors: [ + { + isDefaultBehavior: true, + compress: true, + allowedMethods: CloudFrontAllowedMethods.GET_HEAD, + lambdaFunctionAssociations: [{ + eventType: LambdaEdgeEventType.VIEWER_REQUEST, + lambdaFunction: urlRewriter.currentVersion, + }], + defaultTtl: Duration.seconds(300), + }, + ], + }, + ], + }); + + new CfnOutput(stack, 'BuildDistributionDomainName', { + value: distro.distributionDomainName, + description: 'The domain name where the build artifacts will be available', + }); + } +} diff --git a/lib/buildArtifacts/build-artifacts-permissions.ts b/lib/buildArtifacts/build-artifacts-permissions.ts new file mode 100644 index 00000000..02f78f6e --- /dev/null +++ b/lib/buildArtifacts/build-artifacts-permissions.ts @@ -0,0 +1,91 @@ +/** + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +import { Stack } from '@aws-cdk/core'; +import { + ArnPrincipal, Effect, Policy, PolicyStatement, Role, +} from '@aws-cdk/aws-iam'; +import { CiCdnStack } from '../ci-cdn-stack'; + +export interface buildArtifactProps { + mainNodeArn: string, + agentNodeArn: string, + buildBucketArn: string +} + +export class BuildArtifactsPermissions { + private static readonly BUNDLE_ROLE_NAME = 'opensearch-bundle'; + + constructor(stack: CiCdnStack, props: buildArtifactProps) { + const opensearchBundleRole = new Role(stack, BuildArtifactsPermissions.BUNDLE_ROLE_NAME, { + assumedBy: Role.fromRoleArn(stack, 'MainNodeRole', `${props.mainNodeArn}`), + roleName: 'opensearch-bundle', + }); + + opensearchBundleRole.assumeRolePolicy?.addStatements(new PolicyStatement({ + actions: ['sts:AssumeRole'], + principals: [new ArnPrincipal(`${props.agentNodeArn}`)], + })); + + const opensearchBundlePolicies = BuildArtifactsPermissions.getOpensearchBundlePolicies(stack, props.buildBucketArn); + opensearchBundlePolicies.forEach((policy) => { + policy.attachToRole(opensearchBundleRole); + }); + } + + private static getOpensearchBundlePolicies(scope: Stack, s3BucketArn: string): Policy[] { + const policies: Policy[] = []; + const signingPolicy = new Policy(scope, 'signing-policy', { + statements: [ + new PolicyStatement( + { + actions: ['sts:AssumeRole'], + resources: ['arn:aws:iam::447201093745:role/OpenSearchSignerPGPSigning-ArtifactAccessRole'], + effect: Effect.ALLOW, + }, + ), + ], + }); + + const openSearchBundlePolicy = new Policy(scope, 'opensearch-bundle-policy', { + statements: [ + new PolicyStatement({ + actions: [ + 's3:GetObject*', + 's3:ListBucket', + ], + resources: [`${s3BucketArn}`], + effect: Effect.ALLOW, + }), + new PolicyStatement( + { + actions: [ + 's3:GetObject*', + 's3:ListBucket', + 's3:PutObject', + 's3:PutObjectLegalHold', + 's3:PutObjectRetention', + 's3:PutObjectTagging', + 's3:PutObjectVersionTagging', + 's3:Abort*', + ], + resources: [ + `${s3BucketArn}/*/builds/*`, + `${s3BucketArn}/*/shas/*`, + `${s3BucketArn}/*/dist/*`, + `${s3BucketArn}/*/index.json`, + ], + effect: Effect.ALLOW, + }, + ), + ], + }); + policies.push(signingPolicy, openSearchBundlePolicy); + return policies; + } +} diff --git a/lib/ci-cdn-stack.ts b/lib/ci-cdn-stack.ts new file mode 100644 index 00000000..7fb8fd5f --- /dev/null +++ b/lib/ci-cdn-stack.ts @@ -0,0 +1,30 @@ +/** + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +import { + Construct, Fn, Stack, StackProps, +} from '@aws-cdk/core'; +import { BuildArtifactsPermissions } from './buildArtifacts/build-artifacts-permissions'; +import { ArtifactsPublicAccess } from './buildArtifacts/artifacts-public-access'; + +export class CiCdnStack extends Stack { + constructor(scope: Construct, id: string, props: StackProps) { + super(scope, id, props); + const mainNodeRoleArn = Fn.importValue('mainNodeRoleArn'); + const agentNodeRoleArn = Fn.importValue('agentNodeRoleArn'); + const buildBucketArn = Fn.importValue('buildBucketArn'); + + const buildArtifacts = new BuildArtifactsPermissions(this, { + mainNodeArn: mainNodeRoleArn, + agentNodeArn: agentNodeRoleArn, + buildBucketArn, + }); + + const artifactPublicAccess = new ArtifactsPublicAccess(this, buildBucketArn); + } +} diff --git a/lib/ci-config-stack.ts b/lib/ci-config-stack.ts index 07805d5b..5a080248 100644 --- a/lib/ci-config-stack.ts +++ b/lib/ci-config-stack.ts @@ -25,6 +25,8 @@ export class CIConfigStack extends Stack { static readonly OIDC_CONFIGURATION_VALUE_SECRET_EXPORT_VALUE: string = 'OIDCConfigValueSecret'; + static readonly CASC_RELOAD_TOKEN_SECRET_EXPORT_VALUE: string = 'casc'; + constructor(scope: Construct, id: string, props?: StackProps) { super(scope, id, props); @@ -46,6 +48,9 @@ export class CIConfigStack extends Stack { const OIDCConfigValuesSecret = new Secret(this, 'OIDCConfigValues', { description: 'OIDC params in JSON format', }); + const CascReloadTokenValuesSecret = new Secret(this, 'CascReloadTokenValue', { + description: 'Reload token (password) required for configuration as code plugin', + }); new CfnOutput(this, 'certificateArnSecret', { value: arnSecret.secretArn, @@ -76,5 +81,10 @@ export class CIConfigStack extends Stack { value: OIDCConfigValuesSecret.secretArn, exportName: CIConfigStack.OIDC_CONFIGURATION_VALUE_SECRET_EXPORT_VALUE, }); + + new CfnOutput(this, 'cascSecretValue', { + value: CascReloadTokenValuesSecret.secretArn, + exportName: CIConfigStack.CASC_RELOAD_TOKEN_SECRET_EXPORT_VALUE, + }); } } diff --git a/lib/ci-ecr-stack.ts b/lib/ci-ecr-stack.ts deleted file mode 100644 index 4ea25388..00000000 --- a/lib/ci-ecr-stack.ts +++ /dev/null @@ -1,73 +0,0 @@ -import { - Construct, NestedStack, RemovalPolicy, Stack, StackProps, -} from '@aws-cdk/core'; -import { CfnPublicRepository } from '@aws-cdk/aws-ecr'; -import { - ArnPrincipal, IRole, ManagedPolicy, PolicyStatement, Role, -} from '@aws-cdk/aws-iam'; - -export interface EcrStackProps extends StackProps { - /** Should we deploy ECR */ - readonly deployECR?: boolean; - /** Main Node role arn */ - readonly mainNodeAccountNumber: string; - /** removal policy for the ECR Repositories */ - readonly removalPolicy?: RemovalPolicy; - /** list of ecr repositories to be deployed */ - readonly repositories?: string[]; -} - -export class CiEcrStack extends NestedStack { - constructor(scope: Construct, id: string, props: EcrStackProps) { - super(scope, id, props); - - if (props.repositories !== undefined && props.repositories.length > 0) { - CiEcrStack.createRepositories(scope, this, props.removalPolicy ?? RemovalPolicy.RETAIN, props.repositories ?? []); - } - - const ecrPolicy = CiEcrStack.createEcrPolicy(this, id); - - CiEcrStack.createEcrRole(this, ecrPolicy, props.mainNodeAccountNumber); - } - - public static createEcrRole(stack: Stack, ecrPolicy: ManagedPolicy, mainNodeAccountNumber: string) : IRole { - return new Role(stack, 'ecr-stack-role', { - roleName: 'OpenSearch-CI-ECR-ecr-role', - assumedBy: new ArnPrincipal(`arn:aws:iam::${mainNodeAccountNumber}:role/OpenSearch-CI-MainNodeRole`), - managedPolicies: [ - ecrPolicy, - ], - }); - } - - public static createEcrPolicy(stack: Stack, id: String) : ManagedPolicy { - return new ManagedPolicy(stack, 'ecr-policy', - { - description: 'Policy for uploading images to ECR', - managedPolicyName: `${id}-ecr-policy`, - statements: [new PolicyStatement({ - actions: [ - 'ecr-public:BatchCheckLayerAvailability', - 'ecr-public:CompleteLayerUpload', - 'ecr-public:InitiateLayerUpload', - 'ecr-public:PutImage', - 'ecr-public:UploadLayerPart', - ], - resources: ['*'], - }), - new PolicyStatement({ - actions: [ - 'ecr-public:GetAuthorizationToken', - 'sts:GetServiceBearerToken', - ], - resources: ['*'], - })], - }); - } - - public static createRepositories(scope: Construct, stack: Stack, removalPolicy : RemovalPolicy, repositories: string[]) : void { - repositories.map((repoName) => new CfnPublicRepository(scope, repoName, { - repositoryName: repoName, - }).applyRemovalPolicy(removalPolicy)); - } -} diff --git a/lib/ci-stack.ts b/lib/ci-stack.ts index b93fcd57..41696e87 100644 --- a/lib/ci-stack.ts +++ b/lib/ci-stack.ts @@ -9,10 +9,12 @@ import { FlowLogDestination, FlowLogTrafficType, Vpc } from '@aws-cdk/aws-ec2'; import { Secret } from '@aws-cdk/aws-secretsmanager'; import { + CfnOutput, CfnParameter, Construct, Fn, RemovalPolicy, Stack, StackProps, } from '@aws-cdk/core'; import { ListenerCertificate } from '@aws-cdk/aws-elasticloadbalancingv2'; import { FileSystem } from '@aws-cdk/aws-efs'; +import { Bucket } from '@aws-cdk/aws-s3'; import { CIConfigStack } from './ci-config-stack'; import { JenkinsMainNode } from './compute/jenkins-main-node'; import { JenkinsMonitoring } from './monitoring/ci-alarms'; @@ -30,14 +32,18 @@ export interface CIStackProps extends StackProps { readonly runWithOidc?: boolean; /** Additional verification during deployment and resource startup. */ readonly ignoreResourcesFailures?: boolean; - /** Account to deploy your ECR Assets on */ - readonly ecrAccountId?: string; /** Users with admin access during initial deployment */ readonly adminUsers?: string[]; /** Additional logic that needs to be run on Master Node. The value has to be path to a file */ readonly additionalCommands?: string; /** Do you want to retain jenkins jobs and build history */ readonly dataRetention?: boolean; + /** Policy for agent node role to assume a cross-account role */ + readonly agentAssumeRole?: string; + /** File path containing global environment variables to be added to jenkins enviornment */ + readonly envVarsFilePath?: string; + /** Add Mac agent to jenkins */ + readonly macAgent?: boolean; } export class CIStack extends Stack { @@ -56,6 +62,9 @@ export class CIStack extends Stack { }, }); + const agentAssumeRoleContext = `${props?.agentAssumeRole ?? this.node.tryGetContext('agentAssumeRole')}`; + const macAgentParameter = `${props?.macAgent ?? this.node.tryGetContext('macAgent')}`; + const useSslParameter = `${props?.useSsl ?? this.node.tryGetContext('useSsl')}`; if (useSslParameter !== 'true' && useSslParameter !== 'false') { throw new Error('useSsl parameter is required to be set as - true or false'); @@ -92,15 +101,19 @@ export class CIStack extends Stack { const importedRedirectUrlSecretBucketValue = Fn.importValue(`${CIConfigStack.REDIRECT_URL_SECRET_EXPORT_VALUE}`); const importedOidcConfigValuesSecretBucketValue = Fn.importValue(`${CIConfigStack.OIDC_CONFIGURATION_VALUE_SECRET_EXPORT_VALUE}`); const certificateArn = Secret.fromSecretCompleteArn(this, 'certificateArn', importedArnSecretBucketValue.toString()); + const importedReloadPasswordSecretsArn = Fn.importValue(`${CIConfigStack.CASC_RELOAD_TOKEN_SECRET_EXPORT_VALUE}`); const listenerCertificate = ListenerCertificate.fromArn(certificateArn.secretValue.toString()); - const agentNode = new AgentNodes(this); - const agentNodes: AgentNodeProps[] = [agentNode.AL2_X64, agentNode.AL2_ARM64]; + const agentNode = new AgentNodes(); + const agentNodes: AgentNodeProps[] = [agentNode.AL2_X64, agentNode.AL2_X64_DOCKER_HOST, agentNode.AL2_X64_DOCKER_HOST_PERF_TEST, + agentNode.AL2_ARM64, agentNode.AL2_ARM64_DOCKER_HOST, agentNode.UBUNTU_X64, agentNode.UBUNTU_X64_DOCKER_BUILDER]; const mainJenkinsNode = new JenkinsMainNode(this, { vpc, sg: securityGroups.mainNodeSG, efsSG: securityGroups.efsSG, dataRetention: props.dataRetention ?? false, + envVarsFilePath: props.envVarsFilePath ?? '', + reloadPasswordSecretsArn: importedReloadPasswordSecretsArn.toString(), sslCertContentsArn: importedContentsSecretBucketValue.toString(), sslCertChainArn: importedContentsChainBucketValue.toString(), sslCertPrivateKeyContentsArn: importedCertSecretBucketValue.toString(), @@ -109,11 +122,10 @@ export class CIStack extends Stack { useSsl, runWithOidc, failOnCloudInitError: props?.ignoreResourcesFailures, - ecrAccountId: props.ecrAccountId ?? Stack.of(this).account, adminUsers: props?.adminUsers, agentNodeSecurityGroup: securityGroups.agentNodeSG.securityGroupId, subnetId: vpc.publicSubnets[0].subnetId, - }, agentNodes); + }, agentNodes, agentAssumeRoleContext.toString(), macAgentParameter.toString()); const externalLoadBalancer = new JenkinsExternalLoadBalancer(this, { vpc, @@ -123,10 +135,17 @@ export class CIStack extends Stack { useSsl, }); + const artifactBucket = new Bucket(this, 'BuildBucket'); + this.monitoring = new JenkinsMonitoring(this, externalLoadBalancer, mainJenkinsNode); if (additionalCommandsContext.toString() !== 'undefined') { new RunAdditionalCommands(this, additionalCommandsContext.toString()); } + + new CfnOutput(this, 'Artifact Bucket Arn', { + value: artifactBucket.bucketArn.toString(), + exportName: 'buildBucketArn', + }); } } diff --git a/lib/compute/agent-node-config.ts b/lib/compute/agent-node-config.ts index 39208e7e..23b00dfa 100644 --- a/lib/compute/agent-node-config.ts +++ b/lib/compute/agent-node-config.ts @@ -6,108 +6,248 @@ * compatible open source license. */ -import { CfnInstanceProfile, ServicePrincipal, Role } from '@aws-cdk/aws-iam'; -import { Fn, Stack, Tags } from '@aws-cdk/core'; +import { + Role, ManagedPolicy, PolicyStatement, Effect, CfnInstanceProfile, ServicePrincipal, +} from '@aws-cdk/aws-iam'; +import { + Fn, Stack, Tags, CfnOutput, +} from '@aws-cdk/core'; import { KeyPair } from 'cdk-ec2-key-pair'; import { readFileSync } from 'fs'; import { load } from 'js-yaml'; import { JenkinsMainNode } from './jenkins-main-node'; export interface AgentNodeProps { - amiId: string; - instanceType: string; - workerLabelString: string; - remoteUser: string; - initScript: string -} - + amiId: string; + instanceType: string; + workerLabelString: string; + remoteUser: string; + numExecutors: number; + initScript: string + } export interface AgentNodeNetworkProps { - readonly agentNodeSecurityGroup: string; - readonly subnetId: string; -} + readonly agentNodeSecurityGroup: string; + readonly subnetId: string; + } export class AgentNodeConfig { - public readonly AgentNodeInstanceProfileArn: string; - - public readonly STACKREGION: string; - - public readonly SSHEC2KeySecretId: string; - - constructor(stack: Stack) { - this.STACKREGION = stack.region; - const key = new KeyPair(stack, 'AgentNode-KeyPair', { - name: 'AgentNodeKeyPair', - description: 'KeyPair used by Jenkins Main Node to SSH into Agent Nodes', - }); - Tags.of(key) - .add('jenkins:credentials:type', 'sshUserPrivateKey'); - const AgentNodeRole = new Role(stack, 'JenkinsAgentNodeRole', { - assumedBy: new ServicePrincipal('ec2.amazonaws.com'), - }); - const AgentNodeInstanceProfile = new CfnInstanceProfile(stack, 'JenkinsAgentNodeInstanceProfile', { roles: [AgentNodeRole.roleName] }); - this.AgentNodeInstanceProfileArn = AgentNodeInstanceProfile.attrArn.toString(); - this.SSHEC2KeySecretId = Fn.join('/', ['ec2-ssh-key', key.keyPairName.toString(), 'private']); - } - - public addAgentConfigToJenkinsYaml(templates: AgentNodeProps[], props: AgentNodeNetworkProps): any { - const jenkinsYaml: any = load(readFileSync(JenkinsMainNode.BASE_JENKINS_YAML_PATH, 'utf-8')); - const configTemplates: any = []; - - templates.forEach((element) => { - configTemplates.push(this.getTemplate(element, props)); - }); - - const agentNodeYamlConfig = [{ - amazonEC2: { - cloudName: 'Amazon_ec2_cloud', - region: this.STACKREGION, - sshKeysCredentialsId: this.SSHEC2KeySecretId, - templates: configTemplates, - useInstanceProfileForCredentials: true, - }, - }]; - jenkinsYaml.jenkins.clouds = agentNodeYamlConfig; - return jenkinsYaml; - } - - private getTemplate(config: AgentNodeProps, props: AgentNodeNetworkProps): { [x: string]: any; } { - return { - ami: config.amiId, - amiType: - { unixData: { sshPort: '22' } }, - associatePublicIp: false, - connectBySSHProcess: false, - connectionStrategy: 'PRIVATE_IP', - customDeviceMapping: '/dev/xvda=:100:true:::encrypted', - deleteRootOnTermination: true, - description: `jenkinsAgentNode-${config.workerLabelString}`, - ebsEncryptRootVolume: 'ENCRYPTED', - ebsOptimized: false, - hostKeyVerificationStrategy: 'OFF', - iamInstanceProfile: this.AgentNodeInstanceProfileArn, - idleTerminationMinutes: '30', - initScript: config.initScript, - labelString: config.workerLabelString, - launchTimeoutStr: '300', - maxTotalUses: -1, - minimumNumberOfInstances: 0, - minimumNumberOfSpareInstances: 1, - mode: 'EXCLUSIVE', - monitoring: true, - numExecutors: 1, - remoteAdmin: config.remoteUser, - remoteFS: '/var/jenkins', - securityGroups: props.agentNodeSecurityGroup, - stopOnTerminate: false, - subnetId: props.subnetId, - t2Unlimited: false, - tags: [{ - name: 'Name', - value: `jenkinsAgentNode-${config.workerLabelString}`, - }], - tenancy: 'Default', - type: config.instanceType, - useEphemeralDevices: false, - }; - } + public readonly AgentNodeInstanceProfileArn: string; + + public readonly STACKREGION: string; + + public readonly ACCOUNT: string; + + public readonly SSHEC2KeySecretId: string; + + constructor(stack: Stack, assumeRole: string) { + this.STACKREGION = stack.region; + this.ACCOUNT = stack.account; + + const key = new KeyPair(stack, 'AgentNode-KeyPair', { + name: 'AgentNodeKeyPair', + description: 'KeyPair used by Jenkins Main Node to SSH into Agent Nodes', + }); + Tags.of(key) + .add('jenkins:credentials:type', 'sshUserPrivateKey'); + const AgentNodeRole = new Role(stack, 'OpenSearch-CI-AgentNodeRole', { + assumedBy: new ServicePrincipal('ec2.amazonaws.com'), + // assumedBy: new AccountPrincipal(this.ACCOUNT), + description: 'Jenkins agents Node Role', + roleName: 'OpenSearch-CI-AgentNodeRole', + }); + + const ecrManagedPolicy = new ManagedPolicy(stack, 'OpenSearch-CI-AgentNodePolicy', { + description: 'Jenkins agents Node Policy', + managedPolicyName: 'OpenSearch-CI-AgentNodePolicy', + statements: [ + new PolicyStatement({ + effect: Effect.ALLOW, + actions: [ + 'ecr-public:BatchCheckLayerAvailability', + 'ecr-public:GetRepositoryPolicy', + 'ecr-public:DescribeRepositories', + 'ecr-public:DescribeRegistries', + 'ecr-public:DescribeImages', + 'ecr-public:DescribeImageTags', + 'ecr-public:GetRepositoryCatalogData', + 'ecr-public:GetRegistryCatalogData', + 'ecr-public:InitiateLayerUpload', + 'ecr-public:UploadLayerPart', + 'ecr-public:CompleteLayerUpload', + 'ecr-public:PutImage', + ], + resources: [`arn:aws:ecr-public::${this.ACCOUNT}:repository/*`], + conditions: { + StringEquals: { + 'aws:RequestedRegion': this.STACKREGION, + 'aws:PrincipalAccount': [this.ACCOUNT], + }, + }, + }), + ], + roles: [AgentNodeRole], + }); + ecrManagedPolicy.addStatements( + new PolicyStatement({ + actions: [ + 'ecr-public:GetAuthorizationToken', + 'sts:GetServiceBearerToken', + ], + resources: ['*'], + conditions: { + StringEquals: { + 'aws:RequestedRegion': this.STACKREGION, + 'aws:PrincipalAccount': [this.ACCOUNT], + }, + }, + }), + ); + + /* eslint-disable eqeqeq */ + if (assumeRole.toString() !== 'undefined') { + // policy to allow assume role AssumeRole + AgentNodeRole.addToPolicy( + new PolicyStatement({ + resources: [assumeRole], + actions: ['sts:AssumeRole'], + }), + ); + } + AgentNodeRole.addManagedPolicy(ManagedPolicy.fromAwsManagedPolicyName('AmazonSSMManagedInstanceCore')); + const AgentNodeInstanceProfile = new CfnInstanceProfile(stack, 'JenkinsAgentNodeInstanceProfile', { roles: [AgentNodeRole.roleName] }); + this.AgentNodeInstanceProfileArn = AgentNodeInstanceProfile.attrArn.toString(); + this.SSHEC2KeySecretId = Fn.join('/', ['ec2-ssh-key', key.keyPairName.toString(), 'private']); + + new CfnOutput(stack, 'Jenkins Agent Node Role Arn', { + value: `${AgentNodeRole.roleArn}`, + exportName: 'agentNodeRoleArn', + }); + } + + public addAgentConfigToJenkinsYaml(stack: Stack, templates: AgentNodeProps[], props: AgentNodeNetworkProps, macAgent: string): any { + const jenkinsYaml: any = load(readFileSync(JenkinsMainNode.BASE_JENKINS_YAML_PATH, 'utf-8')); + const configTemplates: any = []; + + templates.forEach((element) => { + configTemplates.push(this.getTemplate(stack, element, props)); + }); + if (macAgent == 'true') { + configTemplates.push(this.getMacTemplate(stack, props)); + } + + const agentNodeYamlConfig = [{ + amazonEC2: { + cloudName: 'Amazon_ec2_cloud', + region: this.STACKREGION, + sshKeysCredentialsId: this.SSHEC2KeySecretId, + templates: configTemplates, + useInstanceProfileForCredentials: true, + }, + }]; + jenkinsYaml.jenkins.clouds = agentNodeYamlConfig; + return jenkinsYaml; + } + + private getTemplate(stack: Stack, config: AgentNodeProps, props: AgentNodeNetworkProps): { [x: string]: any; } { + return { + ami: config.amiId, + amiType: + { unixData: { sshPort: '22' } }, + associatePublicIp: false, + connectBySSHProcess: false, + connectionStrategy: 'PRIVATE_IP', + customDeviceMapping: '/dev/xvda=:300:true:::encrypted', + deleteRootOnTermination: true, + description: `jenkinsAgentNode-${config.workerLabelString}`, + ebsEncryptRootVolume: 'ENCRYPTED', + ebsOptimized: false, + hostKeyVerificationStrategy: 'OFF', + iamInstanceProfile: this.AgentNodeInstanceProfileArn, + idleTerminationMinutes: '60', + initScript: config.initScript, + labelString: config.workerLabelString, + launchTimeoutStr: '300', + maxTotalUses: -1, + minimumNumberOfInstances: 0, + minimumNumberOfSpareInstances: 1, + mode: 'EXCLUSIVE', + monitoring: true, + numExecutors: config.numExecutors, + remoteAdmin: config.remoteUser, + remoteFS: '/var/jenkins', + securityGroups: props.agentNodeSecurityGroup, + stopOnTerminate: false, + subnetId: props.subnetId, + t2Unlimited: false, + tags: [{ + name: 'Name', + value: `${stack.stackName}/AgentNode/${config.workerLabelString}`, + }, + { + name: 'type', + value: `jenkinsAgentNode-${config.workerLabelString}`, + }, + ], + tenancy: 'Default', + type: config.instanceType, + useEphemeralDevices: false, + }; + } + + private getMacTemplate(stack: Stack, props: AgentNodeNetworkProps): { [x: string]: any; } { + return { + ami: 'ami-0379811a08268a97e', + amiType: + { macData: { sshPort: '22' } }, + associatePublicIp: false, + connectBySSHProcess: false, + connectionStrategy: 'PRIVATE_IP', + customDeviceMapping: '/dev/sda1=:300:true:gp3::encrypted', + deleteRootOnTermination: true, + description: 'jenkinsAgentNode-Jenkins-Agent-MacOS-x64-Mac1Metal-Multi-Host', + ebsEncryptRootVolume: 'ENCRYPTED', + ebsOptimized: true, + hostKeyVerificationStrategy: 'OFF', + iamInstanceProfile: this.AgentNodeInstanceProfileArn, + labelString: 'Jenkins-Agent-MacOS-x64-Mac1Metal-Multi-Host', + maxTotalUses: -1, + minimumNumberOfInstances: 1, + minimumNumberOfSpareInstances: 0, + mode: 'EXCLUSIVE', + monitoring: true, + numExecutors: '6', + remoteAdmin: 'ec2-user', + remoteFS: '/var/jenkins', + securityGroups: props.agentNodeSecurityGroup, + stopOnTerminate: false, + subnetId: props.subnetId, + t2Unlimited: false, + tags: [ + { + name: 'Name', + value: `${stack.stackName}/AgentNode/Jenkins-Agent-MacOS-x64-Mac1Metal-Multi-Host`, + }, + { + name: 'type', + value: 'jenkinsAgentNode-Jenkins-Agent-MacOS-x64-Mac1Metal-Multi-Host', + }, + ], + tenancy: 'Host', + type: 'Mac1Metal', + nodeProperties: [ + { + envVars: { + env: [ + { + key: 'Path', + /* eslint-disable max-len */ + value: '/usr/local/opt/python@3.7/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/Cellar/python@3.7/3.7.13_1/Frameworks/Python.framework/Versions/3.7/bin', + }, + ], + }, + }, + ], + useEphemeralDevices: false, + }; + } } diff --git a/lib/compute/agent-nodes.ts b/lib/compute/agent-nodes.ts index 1a511e57..f2c06521 100644 --- a/lib/compute/agent-nodes.ts +++ b/lib/compute/agent-nodes.ts @@ -6,42 +6,87 @@ * compatible open source license. */ -import { - AmazonLinuxCpuType, AmazonLinuxGeneration, MachineImage, -} from '@aws-cdk/aws-ec2'; -import { Stack } from '@aws-cdk/core'; import { AgentNodeProps } from './agent-node-config'; export class AgentNodes { // Refer: https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/ec2/model/InstanceType.html for instance types readonly AL2_X64: AgentNodeProps; + readonly AL2_X64_DOCKER_HOST: AgentNodeProps; + + readonly AL2_X64_DOCKER_HOST_PERF_TEST: AgentNodeProps; + readonly AL2_ARM64: AgentNodeProps; - constructor(stack: Stack) { + readonly AL2_ARM64_DOCKER_HOST: AgentNodeProps; + + readonly UBUNTU_X64: AgentNodeProps; + + readonly UBUNTU_X64_DOCKER_BUILDER: AgentNodeProps; + + constructor() { this.AL2_X64 = { - workerLabelString: 'AL2-X64', + workerLabelString: 'Jenkins-Agent-al2-x64-c54xlarge-Single-Host', + instanceType: 'C54xlarge', + remoteUser: 'ec2-user', + numExecutors: 1, + amiId: 'ami-00a07e55fcad01043', + initScript: 'sudo yum clean all && sudo rm -rf /var/cache/yum/* && sudo yum repolist &&' + + ' sudo yum update --skip-broken --exclude=openssh* --exclude=docker* -y', + }; + this.AL2_X64_DOCKER_HOST = { + workerLabelString: 'Jenkins-Agent-al2-x64-c54xlarge-Docker-Host', instanceType: 'C54xlarge', remoteUser: 'ec2-user', - amiId: MachineImage.latestAmazonLinux({ - generation: AmazonLinuxGeneration.AMAZON_LINUX_2, - cpuType: AmazonLinuxCpuType.X86_64, - }).getImage(stack).imageId.toString(), - initScript: 'sudo mkdir -p /var/jenkins/ && sudo chown -R ec2-user:ec2-user /var/jenkins &&' - + ' sudo yum install -y java-1.8.0-openjdk cmake python3 python3-pip python3-devel && sudo yum groupinstall -y "Development Tools" &&' - + ' sudo ln -sfn `which pip3` /usr/bin/pip && pip3 install pipenv && sudo ln -s ~/.local/bin/pipenv /usr/local/bin', + numExecutors: 8, + amiId: 'ami-00a07e55fcad01043', + initScript: 'sudo yum clean all && sudo rm -rf /var/cache/yum/* && sudo yum repolist &&' + + ' sudo yum update --skip-broken --exclude=openssh* --exclude=docker* -y', + }; + this.AL2_X64_DOCKER_HOST_PERF_TEST = { + workerLabelString: 'Jenkins-Agent-al2-x64-m52xlarge-Docker-Host-Perf-Test', + instanceType: 'M52xlarge', + remoteUser: 'ec2-user', + numExecutors: 8, + amiId: 'ami-00a07e55fcad01043', + initScript: 'sudo yum clean all && sudo rm -rf /var/cache/yum/* && sudo yum repolist &&' + + ' sudo yum update --skip-broken --exclude=openssh* --exclude=docker* -y', }; this.AL2_ARM64 = { - workerLabelString: 'AL2-ARM64', + workerLabelString: 'Jenkins-Agent-al2-arm64-c6g4xlarge-Single-Host', instanceType: 'C6g4xlarge', remoteUser: 'ec2-user', - amiId: MachineImage.latestAmazonLinux({ - generation: AmazonLinuxGeneration.AMAZON_LINUX_2, - cpuType: AmazonLinuxCpuType.ARM_64, - }).getImage(stack).imageId.toString(), - initScript: 'sudo mkdir -p /var/jenkins/ && sudo chown -R ec2-user:ec2-user /var/jenkins &&' - + ' sudo yum install -y java-1.8.0-openjdk cmake python3 python3-pip python3-devel && sudo yum groupinstall -y "Development Tools" &&' - + ' sudo ln -sfn `which pip3` /usr/bin/pip && pip3 install pipenv && sudo ln -s ~/.local/bin/pipenv /usr/local/bin', + numExecutors: 1, + amiId: 'ami-020c52efb1a60f1ae', + initScript: 'sudo yum clean all && sudo rm -rf /var/cache/yum/* && sudo yum repolist &&' + + ' sudo yum update --skip-broken --exclude=openssh* --exclude=docker* -y', + }; + this.AL2_ARM64_DOCKER_HOST = { + workerLabelString: 'Jenkins-Agent-al2-arm64-c6g4xlarge-Docker-Host', + instanceType: 'C6g4xlarge', + remoteUser: 'ec2-user', + numExecutors: 8, + amiId: 'ami-020c52efb1a60f1ae', + initScript: 'sudo yum clean all && sudo rm -rf /var/cache/yum/* && sudo yum repolist &&' + + ' sudo yum update --skip-broken --exclude=openssh* --exclude=docker* -y', + }; + this.UBUNTU_X64 = { + workerLabelString: 'Jenkins-Agent-Ubuntu2004-X64-c524xlarge-Single-Host', + instanceType: 'C524xlarge', + remoteUser: 'ubuntu', + numExecutors: 1, + amiId: 'ami-0f6ceb3b3687a3fba', + initScript: 'sudo apt-mark hold docker docker.io openssh-server && docker ps &&' + + ' sudo apt-get update -y && sudo apt-get upgrade -y && sudo apt-get install docker-compose -y', + }; + this.UBUNTU_X64_DOCKER_BUILDER = { + workerLabelString: 'Jenkins-Agent-Ubuntu2004-X64-m52xlarge-Docker-Builder', + instanceType: 'M52xlarge', + remoteUser: 'ubuntu', + numExecutors: 1, + amiId: 'ami-0f6ceb3b3687a3fba', + initScript: 'sudo apt-mark hold docker docker.io openssh-server && docker ps &&' + + ' sudo apt-get update -y && sudo apt-get upgrade -y', }; } } diff --git a/lib/compute/env-config.ts b/lib/compute/env-config.ts new file mode 100644 index 00000000..58d016cb --- /dev/null +++ b/lib/compute/env-config.ts @@ -0,0 +1,39 @@ +import { readFileSync } from 'fs'; +import { load } from 'js-yaml'; + +/** + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +export class Env { + public readonly key: string + + public readonly value: string + + constructor(key : string, value: string) { + this.key = key; + this.value = value; + } +} + +export class EnvConfig { + public static addEnvConfigToJenkinsYaml(yamlObject: any, envVarsFilePath: string): any { + const jenkinsYaml: any = yamlObject; + const envArray: Env[] = []; + const envFile: any = load(readFileSync(envVarsFilePath, 'utf-8')); + + Object.keys(envFile).forEach((item) => envArray.push(new Env(item, envFile[item]))); + const newEnvVars: Env[] = envArray; + + const envConfig: { [x: string]: any; } = { + envVars: { + env: newEnvVars, + }, + }; + jenkinsYaml.jenkins.globalNodeProperties = [envConfig]; + return jenkinsYaml; + } +} diff --git a/lib/compute/jenkins-main-node.ts b/lib/compute/jenkins-main-node.ts index cffa6a00..4baf0d59 100644 --- a/lib/compute/jenkins-main-node.ts +++ b/lib/compute/jenkins-main-node.ts @@ -6,7 +6,7 @@ * compatible open source license. */ -import { Duration, Stack } from '@aws-cdk/core'; +import { CfnOutput, Duration, Stack } from '@aws-cdk/core'; import { AmazonLinuxGeneration, BlockDeviceVolume, @@ -25,7 +25,7 @@ import { Vpc, } from '@aws-cdk/aws-ec2'; import { - Effect, IManagedPolicy, ManagedPolicy, PolicyStatement, Role, ServicePrincipal, + IManagedPolicy, ManagedPolicy, PolicyStatement, Role, ServicePrincipal, } from '@aws-cdk/aws-iam'; import { Metric, Unit } from '@aws-cdk/aws-cloudwatch'; import { join } from 'path'; @@ -35,6 +35,7 @@ import { FileSystem, PerformanceMode, ThroughputMode } from '@aws-cdk/aws-efs'; import { OidcConfig } from './oidc-config'; import { AgentNodeConfig, AgentNodeNetworkProps, AgentNodeProps } from './agent-node-config'; import { CloudwatchAgent } from '../constructs/cloudwatch-agent'; +import { EnvConfig } from './env-config'; interface HttpConfigProps { readonly redirectUrlArn: string; @@ -50,18 +51,16 @@ interface OidcFederateProps { readonly adminUsers?: string[]; } -interface EcrStackProps { - readonly ecrAccountId: string -} - interface DataRetentionProps { readonly dataRetention?: boolean; readonly efsSG?: SecurityGroup; } -export interface JenkinsMainNodeProps extends HttpConfigProps, OidcFederateProps, EcrStackProps, AgentNodeNetworkProps, DataRetentionProps{ +export interface JenkinsMainNodeProps extends HttpConfigProps, OidcFederateProps, AgentNodeNetworkProps, DataRetentionProps{ readonly vpc: Vpc; readonly sg: SecurityGroup; + readonly envVarsFilePath: string; + readonly reloadPasswordSecretsArn: string; readonly failOnCloudInitError?: boolean; } @@ -80,6 +79,10 @@ export class JenkinsMainNode { private readonly EFS_ID: string; + private static ACCOUNT: string; + + private static STACKREGION: string + public readonly ec2Instance: Instance; public readonly ec2InstanceMetrics: { @@ -88,7 +91,7 @@ export class JenkinsMainNode { foundJenkinsProcessCount: Metric } - constructor(stack: Stack, props: JenkinsMainNodeProps, agentNode: AgentNodeProps[]) { + constructor(stack: Stack, props: JenkinsMainNodeProps, agentNode: AgentNodeProps[], assumeRole: string, macAgent: string) { this.ec2InstanceMetrics = { cpuTime: new Metric({ metricName: 'procstat_cpu_usage', @@ -104,8 +107,8 @@ export class JenkinsMainNode { }), }; - const agentNodeConfig = new AgentNodeConfig(stack); - const jenkinsyaml = JenkinsMainNode.addConfigtoJenkinsYaml(stack, props, agentNodeConfig, props, agentNode); + const agentNodeConfig = new AgentNodeConfig(stack, assumeRole); + const jenkinsyaml = JenkinsMainNode.addConfigtoJenkinsYaml(stack, props, props, agentNodeConfig, props, agentNode, macAgent); if (props.dataRetention) { const efs = new FileSystem(stack, 'EFSfilesystem', { vpc: props.vpc, @@ -142,6 +145,7 @@ export class JenkinsMainNode { props, props, jenkinsyaml, + props.reloadPasswordSecretsArn, this.EFS_ID, )), blockDevices: [{ @@ -150,13 +154,19 @@ export class JenkinsMainNode { }], }); - JenkinsMainNode.createPoliciesForMainNode(stack, props).map( + JenkinsMainNode.createPoliciesForMainNode(stack).map( (policy) => this.ec2Instance.role.addManagedPolicy(policy), ); + + new CfnOutput(stack, 'Jenkins Main Node Role Arn', { + value: this.ec2Instance.role.roleArn, + exportName: 'mainNodeRoleArn', + }); } - public static createPoliciesForMainNode(stack: Stack, ecrStackProps: EcrStackProps): (IManagedPolicy | ManagedPolicy)[] { - const policies: (IManagedPolicy | ManagedPolicy)[] = []; + public static createPoliciesForMainNode(stack: Stack): (IManagedPolicy | ManagedPolicy)[] { + this.STACKREGION = stack.region; + this.ACCOUNT = stack.account; // Policy for SSM management of the host - Removes the need of SSH keys const ec2SsmManagementPolicy = ManagedPolicy.fromAwsManagedPolicyName('AmazonSSMManagedInstanceCore'); @@ -201,23 +211,21 @@ export class JenkinsMainNode { 'ec2:DescribeAvailabilityZones', ], resources: ['*'], + conditions: { + 'ForAllValues:StringEquals': { + 'aws:RequestedRegion': this.STACKREGION, + 'aws:PrincipalAccount': this.ACCOUNT, + }, + }, })], }); - const ecrPolicy = new ManagedPolicy(stack, 'ecr-policy-main-node', { - description: 'Policy for ECR to assume role', - statements: [new PolicyStatement({ - actions: ['sts:AssumeRole'], - effect: Effect.ALLOW, - resources: [`arn:aws:iam::${ecrStackProps.ecrAccountId}:role/OpenSearch-CI-ECR-ecr-role`], - })], - }); - - return [ec2SsmManagementPolicy, cloudwatchEventPublishingPolicy, accessPolicy, mainJenkinsNodePolicy, ecrPolicy]; + return [ec2SsmManagementPolicy, cloudwatchEventPublishingPolicy, accessPolicy, mainJenkinsNodePolicy]; } public static configElements(stackName: string, stackRegion: string, httpConfigProps: HttpConfigProps, - oidcFederateProps: OidcFederateProps, dataRetentionProps : DataRetentionProps, jenkinsyaml: string, efsId?: string): InitElement[] { + oidcFederateProps: OidcFederateProps, dataRetentionProps : DataRetentionProps, jenkinsyaml: string, + reloadPasswordSecretsArn: string, efsId?: string): InitElement[] { return [ InitPackage.yum('wget'), InitPackage.yum('openssl'), @@ -360,15 +368,15 @@ export class JenkinsMainNode { : 'echo Data rentention is disabled, not mounting efs'), InitFile.fromFileInline('/docker-compose.yml', join(__dirname, '../../resources/docker-compose.yml')), - InitCommand.shellCommand('systemctl start docker && docker-compose up -d'), + + InitCommand.shellCommand('systemctl start docker &&' + + ` var=\`aws --region ${stackRegion} secretsmanager get-secret-value --secret-id ${reloadPasswordSecretsArn} --query SecretString --output text\` &&` + + ' yq -i \'.services.jenkins.environment[1] = "CASC_RELOAD_TOKEN=\'$var\'"\' docker-compose.yml &&' + + ' docker-compose up -d'), // Commands are fired one after the other but it does not wait for the command to complete. // Therefore, sleep 90 seconds to wait for jenkins to start - InitCommand.shellCommand('sleep 90'), - - // Download jenkins-cli from the local machine - InitCommand.shellCommand('until $(curl --output /dev/null --silent --head --fail http://localhost:8080); do sleep 5; done &&' - + ' wget -O "jenkins-cli.jar" http://localhost:8080/jnlpJars/jenkins-cli.jar'), + InitCommand.shellCommand('sleep 60'), InitFile.fromFileInline('/initial_jenkins.yaml', jenkinsyaml), @@ -385,18 +393,22 @@ export class JenkinsMainNode { // Reload configuration via Jenkins.yaml InitCommand.shellCommand('cp /initial_jenkins.yaml /var/lib/jenkins/jenkins.yaml &&' - + ' java -jar /jenkins-cli.jar -s http://localhost:8080 reload-jcasc-configuration'), - + + ` var=\`aws --region ${stackRegion} secretsmanager get-secret-value --secret-id ${reloadPasswordSecretsArn} --query SecretString --output text\` &&` + + ' curl -f -X POST "http://localhost:8080/reload-configuration-as-code/?casc-reload-token=$var"'), ]; } - public static addConfigtoJenkinsYaml(stack: Stack, oidcProps: OidcFederateProps, agentNodeObject: AgentNodeConfig, - props: AgentNodeNetworkProps, agentNode: AgentNodeProps[]): string { - let updatedConfig = agentNodeObject.addAgentConfigToJenkinsYaml(agentNode, props); + public static addConfigtoJenkinsYaml(stack: Stack, jenkinsMainNodeProps:JenkinsMainNodeProps, oidcProps: OidcFederateProps, agentNodeObject: AgentNodeConfig, + props: AgentNodeNetworkProps, agentNode: AgentNodeProps[], macAgent: string): string { + let updatedConfig = agentNodeObject.addAgentConfigToJenkinsYaml(stack, agentNode, props, macAgent); if (oidcProps.runWithOidc) { updatedConfig = OidcConfig.addOidcConfigToJenkinsYaml(updatedConfig, oidcProps.adminUsers); } + if (jenkinsMainNodeProps.envVarsFilePath !== '' && jenkinsMainNodeProps.envVarsFilePath != null) { + updatedConfig = EnvConfig.addEnvConfigToJenkinsYaml(updatedConfig, jenkinsMainNodeProps.envVarsFilePath); + } const newConfig = dump(updatedConfig); + writeFileSync(JenkinsMainNode.NEW_JENKINS_YAML_PATH, newConfig, 'utf-8'); return JenkinsMainNode.NEW_JENKINS_YAML_PATH; } diff --git a/lib/compute/oidc-config.ts b/lib/compute/oidc-config.ts index 8e75f485..3f459e88 100644 --- a/lib/compute/oidc-config.ts +++ b/lib/compute/oidc-config.ts @@ -7,7 +7,7 @@ */ export class OidcConfig { - public static readonly adminRolePermissions: string[] = [ + private static readonly adminRolePermissions: string[] = [ 'Overall/Administer', 'Overall/Read', 'Job/Move', @@ -20,9 +20,6 @@ export class OidcConfig { 'Job/Configure', 'Job Config History/DeleteEntry', 'Job/Workspace', - 'Lockable Resources/View', - 'Lockable Resources/Unlock', - 'Lockable Resources/Reserve', 'Credentials/Delete', 'Credentials/ManageDomains', 'Credentials/Update', @@ -47,7 +44,7 @@ export class OidcConfig { 'SCM/Tag', ]; - public static readonly readOnlyRolePermissions: string[] = [ + private static readonly readOnlyRolePermissions: string[] = [ 'Overall/Read', 'Job/Read', ]; diff --git a/lib/deploy-aws-assets.ts b/lib/deploy-aws-assets.ts deleted file mode 100644 index f8ad2466..00000000 --- a/lib/deploy-aws-assets.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { - CfnParameter, Construct, Stack, -} from '@aws-cdk/core'; -import { CiEcrStack, EcrStackProps } from './ci-ecr-stack'; - -export interface DeployAssetsProps extends EcrStackProps { - /** Environment in which assets are being deployed */ - readonly envName: string; -} - -export class DeployAwsAssets extends Stack { - constructor(scope: Construct, id: string, props: DeployAssetsProps) { - super(scope, id, props); - const deployECRParameter = `${props?.deployECR ?? this.node.tryGetContext('deployEcr')}`; - - new CfnParameter(this, 'deployEcr', { - description: 'Should ECR repositories + roles be created', - default: deployECRParameter === 'true', - }); - - if (deployECRParameter === 'true') { - DeployAwsAssets.deployEcrStack(this, props); - } - } - - public static deployEcrStack(scope: Construct, props: DeployAssetsProps): void { - new CiEcrStack(scope, `OpenSearch-CI-ECR-${props.envName}`, props); - } -} diff --git a/lib/network/ci-external-load-balancer.ts b/lib/network/ci-external-load-balancer.ts index 4d642815..548705fb 100644 --- a/lib/network/ci-external-load-balancer.ts +++ b/lib/network/ci-external-load-balancer.ts @@ -42,7 +42,7 @@ export class JenkinsExternalLoadBalancer { const accessPort = props.useSsl ? 443 : 80; this.listener = this.loadBalancer.addListener('JenkinsListener', { - sslPolicy: props.useSsl ? SslPolicy.TLS12 : undefined, + sslPolicy: props.useSsl ? SslPolicy.RECOMMENDED : undefined, port: accessPort, open: true, certificates: props.useSsl ? [props.listenerCertificate] : undefined, diff --git a/package-lock.json b/package-lock.json index deda4447..f090fcee 100644 --- a/package-lock.json +++ b/package-lock.json @@ -59,6 +59,34 @@ "@aws-cdk/aws-route53": "1.127.0", "@aws-cdk/core": "1.127.0", "constructs": "^3.3.69" + }, + "dependencies": { + "@aws-cdk/aws-lambda": { + "version": "1.127.0", + "resolved": "https://registry.npmjs.org/@aws-cdk/aws-lambda/-/aws-lambda-1.127.0.tgz", + "integrity": "sha512-OPYaBYDqPyuqOY9Q/UWMiNSGu3envS5Gn6qEc0O6sW7s1GKgVNNbPBn+LDeeIbp+inwdCNOSFQRy/hTIfQctVg==", + "requires": { + "@aws-cdk/aws-applicationautoscaling": "1.127.0", + "@aws-cdk/aws-cloudwatch": "1.127.0", + "@aws-cdk/aws-codeguruprofiler": "1.127.0", + "@aws-cdk/aws-ec2": "1.127.0", + "@aws-cdk/aws-ecr": "1.127.0", + "@aws-cdk/aws-ecr-assets": "1.127.0", + "@aws-cdk/aws-efs": "1.127.0", + "@aws-cdk/aws-events": "1.127.0", + "@aws-cdk/aws-iam": "1.127.0", + "@aws-cdk/aws-kms": "1.127.0", + "@aws-cdk/aws-logs": "1.127.0", + "@aws-cdk/aws-s3": "1.127.0", + "@aws-cdk/aws-s3-assets": "1.127.0", + "@aws-cdk/aws-signer": "1.127.0", + "@aws-cdk/aws-sqs": "1.127.0", + "@aws-cdk/core": "1.127.0", + "@aws-cdk/cx-api": "1.127.0", + "@aws-cdk/region-info": "1.127.0", + "constructs": "^3.3.69" + } + } } }, "@aws-cdk/aws-cloudformation": { @@ -73,6 +101,80 @@ "@aws-cdk/core": "1.127.0", "@aws-cdk/cx-api": "1.127.0", "constructs": "^3.3.69" + }, + "dependencies": { + "@aws-cdk/aws-lambda": { + "version": "1.127.0", + "resolved": "https://registry.npmjs.org/@aws-cdk/aws-lambda/-/aws-lambda-1.127.0.tgz", + "integrity": "sha512-OPYaBYDqPyuqOY9Q/UWMiNSGu3envS5Gn6qEc0O6sW7s1GKgVNNbPBn+LDeeIbp+inwdCNOSFQRy/hTIfQctVg==", + "requires": { + "@aws-cdk/aws-applicationautoscaling": "1.127.0", + "@aws-cdk/aws-cloudwatch": "1.127.0", + "@aws-cdk/aws-codeguruprofiler": "1.127.0", + "@aws-cdk/aws-ec2": "1.127.0", + "@aws-cdk/aws-ecr": "1.127.0", + "@aws-cdk/aws-ecr-assets": "1.127.0", + "@aws-cdk/aws-efs": "1.127.0", + "@aws-cdk/aws-events": "1.127.0", + "@aws-cdk/aws-iam": "1.127.0", + "@aws-cdk/aws-kms": "1.127.0", + "@aws-cdk/aws-logs": "1.127.0", + "@aws-cdk/aws-s3": "1.127.0", + "@aws-cdk/aws-s3-assets": "1.127.0", + "@aws-cdk/aws-signer": "1.127.0", + "@aws-cdk/aws-sqs": "1.127.0", + "@aws-cdk/core": "1.127.0", + "@aws-cdk/cx-api": "1.127.0", + "@aws-cdk/region-info": "1.127.0", + "constructs": "^3.3.69" + } + } + } + }, + "@aws-cdk/aws-cloudfront": { + "version": "1.127.0", + "resolved": "https://registry.npmjs.org/@aws-cdk/aws-cloudfront/-/aws-cloudfront-1.127.0.tgz", + "integrity": "sha512-upTPNjqJbKtnYzsz3f5PH9bup4OfcHxiR9FSc5ohupMw0gRa9D5GJt2vSOJXdNAlkHx7kmUl6JlipyqzlXQJnA==", + "requires": { + "@aws-cdk/aws-certificatemanager": "1.127.0", + "@aws-cdk/aws-cloudwatch": "1.127.0", + "@aws-cdk/aws-ec2": "1.127.0", + "@aws-cdk/aws-iam": "1.127.0", + "@aws-cdk/aws-kms": "1.127.0", + "@aws-cdk/aws-lambda": "1.127.0", + "@aws-cdk/aws-s3": "1.127.0", + "@aws-cdk/aws-ssm": "1.127.0", + "@aws-cdk/core": "1.127.0", + "@aws-cdk/cx-api": "1.127.0", + "constructs": "^3.3.69" + }, + "dependencies": { + "@aws-cdk/aws-lambda": { + "version": "1.127.0", + "resolved": "https://registry.npmjs.org/@aws-cdk/aws-lambda/-/aws-lambda-1.127.0.tgz", + "integrity": "sha512-OPYaBYDqPyuqOY9Q/UWMiNSGu3envS5Gn6qEc0O6sW7s1GKgVNNbPBn+LDeeIbp+inwdCNOSFQRy/hTIfQctVg==", + "requires": { + "@aws-cdk/aws-applicationautoscaling": "1.127.0", + "@aws-cdk/aws-cloudwatch": "1.127.0", + "@aws-cdk/aws-codeguruprofiler": "1.127.0", + "@aws-cdk/aws-ec2": "1.127.0", + "@aws-cdk/aws-ecr": "1.127.0", + "@aws-cdk/aws-ecr-assets": "1.127.0", + "@aws-cdk/aws-efs": "1.127.0", + "@aws-cdk/aws-events": "1.127.0", + "@aws-cdk/aws-iam": "1.127.0", + "@aws-cdk/aws-kms": "1.127.0", + "@aws-cdk/aws-logs": "1.127.0", + "@aws-cdk/aws-s3": "1.127.0", + "@aws-cdk/aws-s3-assets": "1.127.0", + "@aws-cdk/aws-signer": "1.127.0", + "@aws-cdk/aws-sqs": "1.127.0", + "@aws-cdk/core": "1.127.0", + "@aws-cdk/cx-api": "1.127.0", + "@aws-cdk/region-info": "1.127.0", + "constructs": "^3.3.69" + } + } } }, "@aws-cdk/aws-cloudwatch": { @@ -204,6 +306,34 @@ "@aws-cdk/cx-api": "1.127.0", "@aws-cdk/region-info": "1.127.0", "constructs": "^3.3.69" + }, + "dependencies": { + "@aws-cdk/aws-lambda": { + "version": "1.127.0", + "resolved": "https://registry.npmjs.org/@aws-cdk/aws-lambda/-/aws-lambda-1.127.0.tgz", + "integrity": "sha512-OPYaBYDqPyuqOY9Q/UWMiNSGu3envS5Gn6qEc0O6sW7s1GKgVNNbPBn+LDeeIbp+inwdCNOSFQRy/hTIfQctVg==", + "requires": { + "@aws-cdk/aws-applicationautoscaling": "1.127.0", + "@aws-cdk/aws-cloudwatch": "1.127.0", + "@aws-cdk/aws-codeguruprofiler": "1.127.0", + "@aws-cdk/aws-ec2": "1.127.0", + "@aws-cdk/aws-ecr": "1.127.0", + "@aws-cdk/aws-ecr-assets": "1.127.0", + "@aws-cdk/aws-efs": "1.127.0", + "@aws-cdk/aws-events": "1.127.0", + "@aws-cdk/aws-iam": "1.127.0", + "@aws-cdk/aws-kms": "1.127.0", + "@aws-cdk/aws-logs": "1.127.0", + "@aws-cdk/aws-s3": "1.127.0", + "@aws-cdk/aws-s3-assets": "1.127.0", + "@aws-cdk/aws-signer": "1.127.0", + "@aws-cdk/aws-sqs": "1.127.0", + "@aws-cdk/core": "1.127.0", + "@aws-cdk/cx-api": "1.127.0", + "@aws-cdk/region-info": "1.127.0", + "constructs": "^3.3.69" + } + } } }, "@aws-cdk/aws-elasticloadbalancingv2-targets": { @@ -217,6 +347,34 @@ "@aws-cdk/aws-lambda": "1.127.0", "@aws-cdk/core": "1.127.0", "constructs": "^3.3.69" + }, + "dependencies": { + "@aws-cdk/aws-lambda": { + "version": "1.127.0", + "resolved": "https://registry.npmjs.org/@aws-cdk/aws-lambda/-/aws-lambda-1.127.0.tgz", + "integrity": "sha512-OPYaBYDqPyuqOY9Q/UWMiNSGu3envS5Gn6qEc0O6sW7s1GKgVNNbPBn+LDeeIbp+inwdCNOSFQRy/hTIfQctVg==", + "requires": { + "@aws-cdk/aws-applicationautoscaling": "1.127.0", + "@aws-cdk/aws-cloudwatch": "1.127.0", + "@aws-cdk/aws-codeguruprofiler": "1.127.0", + "@aws-cdk/aws-ec2": "1.127.0", + "@aws-cdk/aws-ecr": "1.127.0", + "@aws-cdk/aws-ecr-assets": "1.127.0", + "@aws-cdk/aws-efs": "1.127.0", + "@aws-cdk/aws-events": "1.127.0", + "@aws-cdk/aws-iam": "1.127.0", + "@aws-cdk/aws-kms": "1.127.0", + "@aws-cdk/aws-logs": "1.127.0", + "@aws-cdk/aws-s3": "1.127.0", + "@aws-cdk/aws-s3-assets": "1.127.0", + "@aws-cdk/aws-signer": "1.127.0", + "@aws-cdk/aws-sqs": "1.127.0", + "@aws-cdk/core": "1.127.0", + "@aws-cdk/cx-api": "1.127.0", + "@aws-cdk/region-info": "1.127.0", + "constructs": "^3.3.69" + } + } } }, "@aws-cdk/aws-events": { @@ -277,6 +435,16 @@ "constructs": "^3.3.69" } }, + "@aws-cdk/aws-lambda-nodejs": { + "version": "1.127.0", + "resolved": "https://registry.npmjs.org/@aws-cdk/aws-lambda-nodejs/-/aws-lambda-nodejs-1.127.0.tgz", + "integrity": "sha512-isPzzg5BprnzYcPN+DnmvugC4sXw7SktMRk5IlmP4+binmNkPUJ8dANg/G3UTYagek5HvEOJShI4s81K/b2wLw==", + "requires": { + "@aws-cdk/aws-lambda": "1.127.0", + "@aws-cdk/core": "1.127.0", + "constructs": "^3.3.69" + } + }, "@aws-cdk/aws-logs": { "version": "1.127.0", "resolved": "https://registry.npmjs.org/@aws-cdk/aws-logs/-/aws-logs-1.127.0.tgz", @@ -353,6 +521,34 @@ "@aws-cdk/core": "1.127.0", "@aws-cdk/cx-api": "1.127.0", "constructs": "^3.3.69" + }, + "dependencies": { + "@aws-cdk/aws-lambda": { + "version": "1.127.0", + "resolved": "https://registry.npmjs.org/@aws-cdk/aws-lambda/-/aws-lambda-1.127.0.tgz", + "integrity": "sha512-OPYaBYDqPyuqOY9Q/UWMiNSGu3envS5Gn6qEc0O6sW7s1GKgVNNbPBn+LDeeIbp+inwdCNOSFQRy/hTIfQctVg==", + "requires": { + "@aws-cdk/aws-applicationautoscaling": "1.127.0", + "@aws-cdk/aws-cloudwatch": "1.127.0", + "@aws-cdk/aws-codeguruprofiler": "1.127.0", + "@aws-cdk/aws-ec2": "1.127.0", + "@aws-cdk/aws-ecr": "1.127.0", + "@aws-cdk/aws-ecr-assets": "1.127.0", + "@aws-cdk/aws-efs": "1.127.0", + "@aws-cdk/aws-events": "1.127.0", + "@aws-cdk/aws-iam": "1.127.0", + "@aws-cdk/aws-kms": "1.127.0", + "@aws-cdk/aws-logs": "1.127.0", + "@aws-cdk/aws-s3": "1.127.0", + "@aws-cdk/aws-s3-assets": "1.127.0", + "@aws-cdk/aws-signer": "1.127.0", + "@aws-cdk/aws-sqs": "1.127.0", + "@aws-cdk/core": "1.127.0", + "@aws-cdk/cx-api": "1.127.0", + "@aws-cdk/region-info": "1.127.0", + "constructs": "^3.3.69" + } + } } }, "@aws-cdk/aws-signer": { @@ -565,6 +761,34 @@ "@aws-cdk/aws-sns": "1.127.0", "@aws-cdk/core": "1.127.0", "constructs": "^3.3.69" + }, + "dependencies": { + "@aws-cdk/aws-lambda": { + "version": "1.127.0", + "resolved": "https://registry.npmjs.org/@aws-cdk/aws-lambda/-/aws-lambda-1.127.0.tgz", + "integrity": "sha512-OPYaBYDqPyuqOY9Q/UWMiNSGu3envS5Gn6qEc0O6sW7s1GKgVNNbPBn+LDeeIbp+inwdCNOSFQRy/hTIfQctVg==", + "requires": { + "@aws-cdk/aws-applicationautoscaling": "1.127.0", + "@aws-cdk/aws-cloudwatch": "1.127.0", + "@aws-cdk/aws-codeguruprofiler": "1.127.0", + "@aws-cdk/aws-ec2": "1.127.0", + "@aws-cdk/aws-ecr": "1.127.0", + "@aws-cdk/aws-ecr-assets": "1.127.0", + "@aws-cdk/aws-efs": "1.127.0", + "@aws-cdk/aws-events": "1.127.0", + "@aws-cdk/aws-iam": "1.127.0", + "@aws-cdk/aws-kms": "1.127.0", + "@aws-cdk/aws-logs": "1.127.0", + "@aws-cdk/aws-s3": "1.127.0", + "@aws-cdk/aws-s3-assets": "1.127.0", + "@aws-cdk/aws-signer": "1.127.0", + "@aws-cdk/aws-sqs": "1.127.0", + "@aws-cdk/core": "1.127.0", + "@aws-cdk/cx-api": "1.127.0", + "@aws-cdk/region-info": "1.127.0", + "constructs": "^3.3.69" + } + } } }, "@aws-cdk/cx-api": { @@ -1969,12 +2193,6 @@ "resolved": "https://registry.npmjs.org/astral-regex/-/astral-regex-2.0.0.tgz", "integrity": "sha512-Z7tMw1ytTXt5jqMcOP+OQteU1VuNK9Y02uuJtKQ1Sv69jXQKKg5cibLwGJow8yzZP+eAc18EmLGPal0bp36rvQ==" }, - "async": { - "version": "3.2.3", - "resolved": "https://registry.npmjs.org/async/-/async-3.2.3.tgz", - "integrity": "sha512-spZRyzKL5l5BZQrr/6m/SqFdBN0q3OCI0f9rjfBzCMBIP4p75P620rR3gTmaksNOhmzgdxcaxdNfMy6anrbM0g==", - "dev": true - }, "asynckit": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", @@ -2191,6 +2409,12 @@ "integrity": "sha512-Z7tMw1ytTXt5jqMcOP+OQteU1VuNK9Y02uuJtKQ1Sv69jXQKKg5cibLwGJow8yzZP+eAc18EmLGPal0bp36rvQ==", "dev": true }, + "async": { + "version": "3.2.1", + "resolved": "https://registry.yarnpkg.com/async/-/async-3.2.1.tgz#d3274ec66d107a47476a4c49136aacdb00665fc8", + "integrity": "sha512-XdD5lRO/87udXCMC9meWdYiR+Nq6ZjUfXidViUZGu2F1MO4T3XwZ1et0hb2++BgLfhyJwy44BGB/yx80ABx8hg==", + "dev": true + }, "at-least-node": { "version": "1.0.0", "resolved": "https://registry.yarnpkg.com/at-least-node/-/at-least-node-1.0.0.tgz#602cd4b46e844ad4effc92a8011a3c46e0238dc2", @@ -2414,6 +2638,12 @@ } } }, + "async": { + "version": "3.2.1", + "resolved": "https://registry.yarnpkg.com/async/-/async-3.2.1.tgz#d3274ec66d107a47476a4c49136aacdb00665fc8", + "integrity": "sha512-XdD5lRO/87udXCMC9meWdYiR+Nq6ZjUfXidViUZGu2F1MO4T3XwZ1et0hb2++BgLfhyJwy44BGB/yx80ABx8hg==", + "dev": true + }, "aws-sdk": { "version": "2.1002.0", "resolved": "https://registry.yarnpkg.com/aws-sdk/-/aws-sdk-2.1002.0.tgz#8f2d3143109a08b19385b21433c9a7178b3f5295", @@ -4137,6 +4367,12 @@ "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", "dev": true }, + "vm2": { + "version": "3.9.3", + "resolved": "https://registry.yarnpkg.com/vm2/-/vm2-3.9.3.tgz#29917f6cc081cc43a3f580c26c5b553fd3c91f40", + "integrity": "sha512-smLS+18RjXYMl9joyJxMNI9l4w7biW8ilSDaVRvFBDwOH8P0BK1ognFQTpg0wyQ6wIKLTblHJvROW692L/E53Q==", + "dev": true + }, "word-wrap": { "version": "1.2.3", "resolved": "https://registry.yarnpkg.com/word-wrap/-/word-wrap-1.2.3.tgz#610636f6b1f703891bd34771ccb17fb93b47079c", @@ -5310,6 +5546,153 @@ "is-symbol": "^1.0.2" } }, + "esbuild": { + "version": "0.14.43", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.14.43.tgz", + "integrity": "sha512-Uf94+kQmy/5jsFwKWiQB4hfo/RkM9Dh7b79p8yqd1tshULdr25G2szLz631NoH3s2ujnKEKVD16RmOxvCNKRFA==", + "requires": { + "esbuild-android-64": "0.14.43", + "esbuild-android-arm64": "0.14.43", + "esbuild-darwin-64": "0.14.43", + "esbuild-darwin-arm64": "0.14.43", + "esbuild-freebsd-64": "0.14.43", + "esbuild-freebsd-arm64": "0.14.43", + "esbuild-linux-32": "0.14.43", + "esbuild-linux-64": "0.14.43", + "esbuild-linux-arm": "0.14.43", + "esbuild-linux-arm64": "0.14.43", + "esbuild-linux-mips64le": "0.14.43", + "esbuild-linux-ppc64le": "0.14.43", + "esbuild-linux-riscv64": "0.14.43", + "esbuild-linux-s390x": "0.14.43", + "esbuild-netbsd-64": "0.14.43", + "esbuild-openbsd-64": "0.14.43", + "esbuild-sunos-64": "0.14.43", + "esbuild-windows-32": "0.14.43", + "esbuild-windows-64": "0.14.43", + "esbuild-windows-arm64": "0.14.43" + } + }, + "esbuild-android-64": { + "version": "0.14.43", + "resolved": "https://registry.npmjs.org/esbuild-android-64/-/esbuild-android-64-0.14.43.tgz", + "integrity": "sha512-kqFXAS72K6cNrB6RiM7YJ5lNvmWRDSlpi7ZuRZ1hu1S3w0zlwcoCxWAyM23LQUyZSs1PbjHgdbbfYAN8IGh6xg==", + "optional": true + }, + "esbuild-android-arm64": { + "version": "0.14.43", + "resolved": "https://registry.npmjs.org/esbuild-android-arm64/-/esbuild-android-arm64-0.14.43.tgz", + "integrity": "sha512-bKS2BBFh+7XZY9rpjiHGRNA7LvWYbZWP87pLehggTG7tTaCDvj8qQGOU/OZSjCSKDYbgY7Q+oDw8RlYQ2Jt2BA==", + "optional": true + }, + "esbuild-darwin-64": { + "version": "0.14.43", + "resolved": "https://registry.npmjs.org/esbuild-darwin-64/-/esbuild-darwin-64-0.14.43.tgz", + "integrity": "sha512-/3PSilx011ttoieRGkSZ0XV8zjBf2C9enV4ScMMbCT4dpx0mFhMOpFnCHkOK0pWGB8LklykFyHrWk2z6DENVUg==", + "optional": true + }, + "esbuild-darwin-arm64": { + "version": "0.14.43", + "resolved": "https://registry.npmjs.org/esbuild-darwin-arm64/-/esbuild-darwin-arm64-0.14.43.tgz", + "integrity": "sha512-1HyFUKs8DMCBOvw1Qxpr5Vv/ThNcVIFb5xgXWK3pyT40WPvgYIiRTwJCvNs4l8i5qWF8/CK5bQxJVDjQvtv0Yw==", + "optional": true + }, + "esbuild-freebsd-64": { + "version": "0.14.43", + "resolved": "https://registry.npmjs.org/esbuild-freebsd-64/-/esbuild-freebsd-64-0.14.43.tgz", + "integrity": "sha512-FNWc05TPHYgaXjbPZO5/rJKSBslfG6BeMSs8GhwnqAKP56eEhvmzwnIz1QcC9cRVyO+IKqWNfmHFkCa1WJTULA==", + "optional": true + }, + "esbuild-freebsd-arm64": { + "version": "0.14.43", + "resolved": "https://registry.npmjs.org/esbuild-freebsd-arm64/-/esbuild-freebsd-arm64-0.14.43.tgz", + "integrity": "sha512-amrYopclz3VohqisOPR6hA3GOWA3LZC1WDLnp21RhNmoERmJ/vLnOpnrG2P/Zao+/erKTCUqmrCIPVtj58DRoA==", + "optional": true + }, + "esbuild-linux-32": { + "version": "0.14.43", + "resolved": "https://registry.npmjs.org/esbuild-linux-32/-/esbuild-linux-32-0.14.43.tgz", + "integrity": "sha512-KoxoEra+9O3AKVvgDFvDkiuddCds6q71owSQEYwjtqRV7RwbPzKxJa6+uyzUulHcyGVq0g15K0oKG5CFBcvYDw==", + "optional": true + }, + "esbuild-linux-64": { + "version": "0.14.43", + "resolved": "https://registry.npmjs.org/esbuild-linux-64/-/esbuild-linux-64-0.14.43.tgz", + "integrity": "sha512-EwINwGMyiJMgBby5/SbMqKcUhS5AYAZ2CpEBzSowsJPNBJEdhkCTtEjk757TN/wxgbu3QklqDM6KghY660QCUw==", + "optional": true + }, + "esbuild-linux-arm": { + "version": "0.14.43", + "resolved": "https://registry.npmjs.org/esbuild-linux-arm/-/esbuild-linux-arm-0.14.43.tgz", + "integrity": "sha512-e6YzQUoDxxtyamuF12eVzzRC7bbEFSZohJ6igQB9tBqnNmIQY3fI6Cns3z2wxtbZ3f2o6idkD2fQnlvs2902Dg==", + "optional": true + }, + "esbuild-linux-arm64": { + "version": "0.14.43", + "resolved": "https://registry.npmjs.org/esbuild-linux-arm64/-/esbuild-linux-arm64-0.14.43.tgz", + "integrity": "sha512-UlSpjMWllAc70zYbHxWuDS3FJytyuR/gHJYBr8BICcTNb/TSOYVBg6U7b3jZ3mILTrgzwJUHwhEwK18FZDouUQ==", + "optional": true + }, + "esbuild-linux-mips64le": { + "version": "0.14.43", + "resolved": "https://registry.npmjs.org/esbuild-linux-mips64le/-/esbuild-linux-mips64le-0.14.43.tgz", + "integrity": "sha512-f+v8cInPEL1/SDP//CfSYzcDNgE4CY3xgDV81DWm3KAPWzhvxARrKxB1Pstf5mB56yAslJDxu7ryBUPX207EZA==", + "optional": true + }, + "esbuild-linux-ppc64le": { + "version": "0.14.43", + "resolved": "https://registry.npmjs.org/esbuild-linux-ppc64le/-/esbuild-linux-ppc64le-0.14.43.tgz", + "integrity": "sha512-5wZYMDGAL/K2pqkdIsW+I4IR41kyfHr/QshJcNpUfK3RjB3VQcPWOaZmc+74rm4ZjVirYrtz+jWw0SgxtxRanA==", + "optional": true + }, + "esbuild-linux-riscv64": { + "version": "0.14.43", + "resolved": "https://registry.npmjs.org/esbuild-linux-riscv64/-/esbuild-linux-riscv64-0.14.43.tgz", + "integrity": "sha512-lYcAOUxp85hC7lSjycJUVSmj4/9oEfSyXjb/ua9bNl8afonaduuqtw7hvKMoKuYnVwOCDw4RSfKpcnIRDWq+Bw==", + "optional": true + }, + "esbuild-linux-s390x": { + "version": "0.14.43", + "resolved": "https://registry.npmjs.org/esbuild-linux-s390x/-/esbuild-linux-s390x-0.14.43.tgz", + "integrity": "sha512-27e43ZhHvhFE4nM7HqtUbMRu37I/4eNSUbb8FGZWszV+uLzMIsHDwLoBiJmw7G9N+hrehNPeQ4F5Ujad0DrUKQ==", + "optional": true + }, + "esbuild-netbsd-64": { + "version": "0.14.43", + "resolved": "https://registry.npmjs.org/esbuild-netbsd-64/-/esbuild-netbsd-64-0.14.43.tgz", + "integrity": "sha512-2mH4QF6hHBn5zzAfxEI/2eBC0mspVsZ6UVo821LpAJKMvLJPBk3XJO5xwg7paDqSqpl7p6IRrAenW999AEfJhQ==", + "optional": true + }, + "esbuild-openbsd-64": { + "version": "0.14.43", + "resolved": "https://registry.npmjs.org/esbuild-openbsd-64/-/esbuild-openbsd-64-0.14.43.tgz", + "integrity": "sha512-ZhQpiZjvqCqO8jKdGp9+8k9E/EHSA+zIWOg+grwZasI9RoblqJ1QiZqqi7jfd6ZrrG1UFBNGe4m0NFxCFbMVbg==", + "optional": true + }, + "esbuild-sunos-64": { + "version": "0.14.43", + "resolved": "https://registry.npmjs.org/esbuild-sunos-64/-/esbuild-sunos-64-0.14.43.tgz", + "integrity": "sha512-DgxSi9DaHReL9gYuul2rrQCAapgnCJkh3LSHPKsY26zytYppG0HgkgVF80zjIlvEsUbGBP/GHQzBtrezj/Zq1Q==", + "optional": true + }, + "esbuild-windows-32": { + "version": "0.14.43", + "resolved": "https://registry.npmjs.org/esbuild-windows-32/-/esbuild-windows-32-0.14.43.tgz", + "integrity": "sha512-Ih3+2O5oExiqm0mY6YYE5dR0o8+AspccQ3vIAtRodwFvhuyGLjb0Hbmzun/F3Lw19nuhPMu3sW2fqIJ5xBxByw==", + "optional": true + }, + "esbuild-windows-64": { + "version": "0.14.43", + "resolved": "https://registry.npmjs.org/esbuild-windows-64/-/esbuild-windows-64-0.14.43.tgz", + "integrity": "sha512-8NsuNfI8xwFuJbrCuI+aBqNTYkrWErejFO5aYM+yHqyHuL8mmepLS9EPzAzk8rvfaJrhN0+RvKWAcymViHOKEw==", + "optional": true + }, + "esbuild-windows-arm64": { + "version": "0.14.43", + "resolved": "https://registry.npmjs.org/esbuild-windows-arm64/-/esbuild-windows-arm64-0.14.43.tgz", + "integrity": "sha512-7ZlD7bo++kVRblJEoG+cepljkfP8bfuTPz5fIXzptwnPaFwGS6ahvfoYzY7WCf5v/1nX2X02HDraVItTgbHnKw==", + "optional": true + }, "escalade": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", @@ -7593,9 +7976,9 @@ } }, "json-schema": { - "version": "0.2.3", - "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.2.3.tgz", - "integrity": "sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM=" + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.4.0.tgz", + "integrity": "sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA==" }, "json-schema-traverse": { "version": "0.4.1", @@ -7631,13 +8014,13 @@ "integrity": "sha1-P02uSpH6wxX3EGL4UhzCOfE2YoA=" }, "jsprim": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.1.tgz", - "integrity": "sha1-MT5mvB5cwG5Di8G3SZwuXFastqI=", + "version": "1.4.2", + "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.2.tgz", + "integrity": "sha512-P2bSOMAc/ciLz6DzgjVlGJP9+BrJWu5UDGK70C2iweC5QBIeFf0ZXRvGjEj2uYgrY2MkAAhsSWHDWlFtEroZWw==", "requires": { "assert-plus": "1.0.0", "extsprintf": "1.3.0", - "json-schema": "0.2.3", + "json-schema": "0.4.0", "verror": "1.10.0" } }, @@ -10340,30 +10723,6 @@ } } }, - "vm2": { - "version": "3.9.9", - "resolved": "https://registry.npmjs.org/vm2/-/vm2-3.9.9.tgz", - "integrity": "sha512-xwTm7NLh/uOjARRBs8/95H0e8fT3Ukw5D/JJWhxMbhKzNh1Nu981jQKvkep9iKYNxzlVrdzD0mlBGkDKZWprlw==", - "dev": true, - "requires": { - "acorn": "^8.7.0", - "acorn-walk": "^8.2.0" - }, - "dependencies": { - "acorn": { - "version": "8.7.0", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.7.0.tgz", - "integrity": "sha512-V/LGr1APy+PXIwKebEWrkZPwoeoF+w1jiOBUmuxuiUIaOHtob8Qc9BTrYo7VuI5fR8tqsy+buA2WFooR5olqvQ==", - "dev": true - }, - "acorn-walk": { - "version": "8.2.0", - "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-8.2.0.tgz", - "integrity": "sha512-k+iyHEuPgSw6SbuDpGQM+06HQUa04DZ3o+F6CSzXMvvI5KMvnaEqXe+YVe555R9nn6GPt404fos4wcgpw12SDA==", - "dev": true - } - } - }, "w3c-hr-time": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/w3c-hr-time/-/w3c-hr-time-1.0.2.tgz", diff --git a/package.json b/package.json index 4ddb8af7..b15fe864 100644 --- a/package.json +++ b/package.json @@ -42,6 +42,10 @@ "source-map-support": "^0.5.16", "@aws-cdk/aws-ecr": "1.127.0", "js-yaml": "^4.1.0", - "@aws-cdk/aws-ssm": "1.127.0" + "@aws-cdk/aws-ssm": "1.127.0", + "@aws-cdk/aws-cloudfront": "1.127.0", + "@aws-cdk/aws-lambda": "1.127.0", + "@aws-cdk/aws-lambda-nodejs": "1.127.0", + "esbuild": "^0.14.43" } } diff --git a/resources/baseJenkins.yaml b/resources/baseJenkins.yaml index 3f0e505b..c6c346fc 100644 --- a/resources/baseJenkins.yaml +++ b/resources/baseJenkins.yaml @@ -8,10 +8,14 @@ jenkins: disableRememberMe: false labelAtoms: - name: "built-in" - markupFormatter: "plainText" - mode: NORMAL + - name: "main-node" + labelString: "main-node" + markupFormatter: + rawHtml: + disableSyntaxHighlighting: true + mode: EXCLUSIVE myViewsTabBar: "standard" - numExecutors: 2 + numExecutors: 4 primaryView: all: name: "all" @@ -58,6 +62,10 @@ unclassified: audit-trail: logBuildCause: true pattern: ".*/(?:configSubmit|doDelete|postBuildResult|enable|disable|cancelQueue|stop|toggleLogKeep|doWipeOutWorkspace|createItem|createView|toggleOffline|cancelQuietDown|quietDown|restart|exit|safeExit)/?.*" + awsCredentialsProvider: + cache: false + client: + credentialsProvider: "default" buildDiscarders: configuredBuildDiscarders: - "jobBuildDiscarder" @@ -149,6 +157,32 @@ tool: installations: - home: "git" name: "Default" + jdk: + installations: + - name: "jdk-8" + properties: + - installSource: + installers: + - adoptOpenJdkInstaller: + id: "jdk8u332-b09" + - name: "jdk-11" + properties: + - installSource: + installers: + - adoptOpenJdkInstaller: + id: "jdk-11.0.15+10" + - name: "jdk-14" + properties: + - installSource: + installers: + - adoptOpenJdkInstaller: + id: "jdk-14.0.2+12" + - name: "jdk-17" + properties: + - installSource: + installers: + - adoptOpenJdkInstaller: + id: "jdk-17.0.3+7" mavenGlobalConfig: globalSettingsProvider: "standard" settingsProvider: "standard" @@ -157,4 +191,4 @@ tool: - home: "powershell.exe" name: "DefaultWindows" - home: "pwsh" - name: "DefaultLinux" \ No newline at end of file + name: "DefaultLinux" diff --git a/resources/cf-url-rewriter/cf-url-rewriter.ts b/resources/cf-url-rewriter/cf-url-rewriter.ts new file mode 100644 index 00000000..cb0d75ee --- /dev/null +++ b/resources/cf-url-rewriter/cf-url-rewriter.ts @@ -0,0 +1,68 @@ +/** + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +import { CloudFrontRequest, CloudFrontRequestCallback, CloudFrontRequestEvent, Context } from 'aws-lambda'; +import { httpsGet } from './https-get'; + +export async function handler(event: CloudFrontRequestEvent, context: Context, callback: CloudFrontRequestCallback) { + const request = event.Records[0].cf.request; + + if (!request.uri.includes('/ci/dbc/')) { + callback(null, errorResponse()); + return; + } + + if (request.uri.includes("/latest/")) { + + const indexUri = request.uri.replace(/\/latest\/.*/, '/index.json'); + + try { + const data: any = await httpsGet('https://' + request.headers.host[0].value + indexUri); + + if (data && data.latest) { + callback(null, redirectResponse(request, data.latest)); + } else { + callback(null, errorResponse()); + } + } catch (e) { + console.log(e); + callback(null, errorResponse()); + } + + } else { + // Incoming URLs from ci.opensearch.org will have a '/ci/123/' prefix, remove the prefix path from requests into S3. + request.uri = request.uri.replace(/^\/ci\/...\//, '\/'); + callback(null, request); + } +} + +function redirectResponse(request: CloudFrontRequest, latestNumber: number) { + return { + status: '302', + statusDescription: 'Moved temporarily', + headers: { + 'location': [{ + key: 'Location', + value: request.uri.replace(/\/latest\//, '/' + latestNumber + '/'), + }], + 'cache-control': [{ + key: 'Cache-Control', + value: "max-age=3600" + }], + }, + }; + +} + +function errorResponse() { + return { + body: 'The page is not found!', + status: '404', + statusDescription: 'Not found', + }; +} diff --git a/resources/cf-url-rewriter/https-get.ts b/resources/cf-url-rewriter/https-get.ts new file mode 100644 index 00000000..16261179 --- /dev/null +++ b/resources/cf-url-rewriter/https-get.ts @@ -0,0 +1,43 @@ +/** + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +import * as https from 'https'; + +export async function httpsGet(url: string) { + + return new Promise((resolve, reject) => { + + const request = https.get(url, (res) => { + + let body = ""; + + res.on("data", (chunk) => { + body += chunk; + }); + + res.on("end", () => { + try { + let json = JSON.parse(body); + resolve(json); + } catch (e) { + console.log(e); + reject({ + error: 'Failed to parse body!' + }); + }; + }); + }); + + request.on("error", (e) => { + console.log(e); + reject({ + error: 'Request error!' + }); + }); + }); +} diff --git a/resources/cf-url-rewriter/package-lock.json b/resources/cf-url-rewriter/package-lock.json new file mode 100644 index 00000000..932a1e50 --- /dev/null +++ b/resources/cf-url-rewriter/package-lock.json @@ -0,0 +1,26 @@ +{ + "name": "cf-url-rewriter", + "version": "1.0.0", + "lockfileVersion": 1, + "requires": true, + "dependencies": { + "@types/aws-lambda": { + "version": "8.10.100", + "resolved": "https://registry.npmjs.org/@types/aws-lambda/-/aws-lambda-8.10.100.tgz", + "integrity": "sha512-Qldp1LhbgQ4tEHewci0/q5qVRaDr8ylvsYfpitJ+FFixzAHpgitvM2NM449uYWt2tYo/iJy25aSRRUVvwIT+ng==", + "dev": true + }, + "@types/node": { + "version": "14.18.21", + "resolved": "https://registry.npmjs.org/@types/node/-/node-14.18.21.tgz", + "integrity": "sha512-x5W9s+8P4XteaxT/jKF0PSb7XEvo5VmqEWgsMlyeY4ZlLK8I6aH6g5TPPyDlLAep+GYf4kefb7HFyc7PAO3m+Q==", + "dev": true + }, + "typescript": { + "version": "4.7.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.7.3.tgz", + "integrity": "sha512-WOkT3XYvrpXx4vMMqlD+8R8R37fZkjyLGlxavMc4iB8lrl8L0DeTcHbYgw/v0N/z9wAFsgBhcsF0ruoySS22mA==", + "dev": true + } + } +} diff --git a/resources/cf-url-rewriter/package.json b/resources/cf-url-rewriter/package.json new file mode 100644 index 00000000..0f95c9aa --- /dev/null +++ b/resources/cf-url-rewriter/package.json @@ -0,0 +1,17 @@ +{ + "name": "cf-url-rewriter", + "version": "1.0.0", + "description": "", + "main": "cf-url-rewriter.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "keywords": [], + "author": "", + "license": "ISC", + "devDependencies": { + "@types/aws-lambda": "^8.10.68", + "@types/node": "^14.14.19", + "typescript": "^4.1.3" + } +} diff --git a/resources/docker-compose.yml b/resources/docker-compose.yml index e2bd424e..3d526b68 100644 --- a/resources/docker-compose.yml +++ b/resources/docker-compose.yml @@ -1,7 +1,8 @@ version: '3.8' services: jenkins: - image: opensearchstaging/jenkins:latest + image: opensearchstaging/jenkins:2.332.3-lts-jdk8 + restart: on-failure privileged: true tty: true user: root @@ -9,12 +10,22 @@ services: - 8080:8080 - 50000:50000 container_name: jenkins + environment: + - JENKINS_JAVA_OPTS="-Xmx8g" + - CASC_RELOAD_TOKEN=reloadPasswordHere volumes: - /var/lib/jenkins:/var/jenkins_home + deploy: + resources: + limits: + cpus: '4' + memory: '10g' + reservations: + cpus: '4' + memory: '10g' logging: driver: awslogs options: awslogs-group: JenkinsMainNode/jenkins.log - awslogs-stream: jenkins.log awslogs-create-group: 'true' - awslogs-datetime-format: '%Y-%m-%d %H:%M:%S.%f%z' \ No newline at end of file + awslogs-datetime-format: '%Y-%m-%d %H:%M:%S.%f%z' diff --git a/test/ci-cdn-stack.test.ts b/test/ci-cdn-stack.test.ts new file mode 100644 index 00000000..19811118 --- /dev/null +++ b/test/ci-cdn-stack.test.ts @@ -0,0 +1,59 @@ +/** + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +import { App } from '@aws-cdk/core'; +import { countResources, expect, haveResourceLike } from '@aws-cdk/assert'; +import { CiCdnStack } from '../lib/ci-cdn-stack'; + +test('CDN Stack Resources', () => { + const cdnApp = new App({ + context: { useSsl: 'true', runWithOidc: 'true', additionalCommands: './test/data/hello-world.py' }, + }); + + // WHEN + const cdnStack = new CiCdnStack(cdnApp, 'cdnTestStack', {}); + + // THEN + expect(cdnStack).to(countResources('AWS::IAM::Role', 2)); + expect(cdnStack).to(countResources('AWS::IAM::Policy', 2)); + expect(cdnStack).to(countResources('AWS::CloudFront::CloudFrontOriginAccessIdentity', 1)); + expect(cdnStack).to(countResources('AWS::CloudFront::Distribution', 1)); + expect(cdnStack).to(haveResourceLike('AWS::CloudFront::Distribution', { + DistributionConfig: { + DefaultCacheBehavior: { + DefaultTTL: 300, + }, + }, + })); + expect(cdnStack).to(countResources('AWS::Lambda::Function', 1)); +}); + +test('CDN Stack Resources With mac agent', () => { + const cdnApp = new App({ + context: { + useSsl: 'true', runWithOidc: 'true', additionalCommands: './test/data/hello-world.py', macAgent: true, + }, + }); + + // WHEN + const cdnStack = new CiCdnStack(cdnApp, 'cdnTestStack', {}); + + // THEN + expect(cdnStack).to(countResources('AWS::IAM::Role', 2)); + expect(cdnStack).to(countResources('AWS::IAM::Policy', 2)); + expect(cdnStack).to(countResources('AWS::CloudFront::CloudFrontOriginAccessIdentity', 1)); + expect(cdnStack).to(countResources('AWS::CloudFront::Distribution', 1)); + expect(cdnStack).to(haveResourceLike('AWS::CloudFront::Distribution', { + DistributionConfig: { + DefaultCacheBehavior: { + DefaultTTL: 300, + }, + }, + })); + expect(cdnStack).to(countResources('AWS::Lambda::Function', 1)); +}); diff --git a/test/ci-ecr-stack.test.ts b/test/ci-ecr-stack.test.ts deleted file mode 100644 index b11dbb04..00000000 --- a/test/ci-ecr-stack.test.ts +++ /dev/null @@ -1,168 +0,0 @@ -/** - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -import { - expect, countResources, haveResourceLike, ResourcePart, -} from '@aws-cdk/assert'; -import { App, RemovalPolicy } from '@aws-cdk/core'; -import { CiEcrStack } from '../lib/ci-ecr-stack'; -import { DeployAssetsProps, DeployAwsAssets } from '../lib/deploy-aws-assets'; - -test('ECR Stack with Resources', () => { - const app = new App({ - context: { useSsl: 'true', runWithOidc: 'true' }, - }); - - // WHEN - const deployAssetsProps : DeployAssetsProps = { - removalPolicy: RemovalPolicy.DESTROY, - mainNodeAccountNumber: '99999999', - envName: 'dev', - env: { - region: 'us-east-1', - }, - deployECR: false, - repositories: [ - 'opensearch', - 'opensearch-dashboards', - ], - }; - - const deployAwsAssetsStack = new DeployAwsAssets(app, 'TestStack', deployAssetsProps); - - const ecrStack = new CiEcrStack(deployAwsAssetsStack, 'ecrStack', deployAssetsProps); - - // THEN - expect(ecrStack).to(countResources('AWS::IAM::Role', 1)); - expect(ecrStack).to(countResources('AWS::IAM::ManagedPolicy', 1)); -}); - -test('ECR Stack No ECR Repositories', () => { - const app = new App({ - context: { useSsl: 'true', runWithOidc: 'true' }, - }); - - // WHEN - const deployAssetsProps : DeployAssetsProps = { - removalPolicy: RemovalPolicy.DESTROY, - mainNodeAccountNumber: '99999999', - envName: 'dev', - env: { - region: 'us-east-1', - }, - deployECR: false, - }; - - const deployAwsAssetsStack = new DeployAwsAssets(app, 'TestStack', deployAssetsProps); - - const ecrStack = new CiEcrStack(deployAwsAssetsStack, 'ecrStack', deployAssetsProps); - - // THEN - expect(ecrStack).to(countResources('AWS::ECR::PublicRepository', 0)); - expect(deployAwsAssetsStack).to(countResources('AWS::ECR::PublicRepository', 0)); - expect(ecrStack).to(countResources('AWS::IAM::Role', 1)); - expect(ecrStack).to(countResources('AWS::IAM::ManagedPolicy', 1)); - expect(deployAwsAssetsStack).to(countResources('AWS::CloudFormation::Stack', 1)); -}); - -test('Test ecr role', () => { - const app = new App({ - context: { useSsl: 'true', runWithOidc: 'true' }, - }); - - // WHEN - const deployAssetsProps : DeployAssetsProps = { - removalPolicy: RemovalPolicy.DESTROY, - mainNodeAccountNumber: '99999999', - - envName: 'dev', - env: { - region: 'us-east-1', - }, - deployECR: false, - }; - - const deployAwsAssetsStack = new DeployAwsAssets(app, 'TestStack', deployAssetsProps); - - const ecrStack = new CiEcrStack(deployAwsAssetsStack, 'ecrStack', deployAssetsProps); - - // THEN - expect(ecrStack).to(haveResourceLike('AWS::IAM::Role', { - AssumeRolePolicyDocument: { - Statement: [ - { - Action: 'sts:AssumeRole', - Effect: 'Allow', - Principal: { - AWS: 'arn:aws:iam::99999999:role/OpenSearch-CI-MainNodeRole', - }, - }, - ], - Version: '2012-10-17', - }, - ManagedPolicyArns: [ - { - Ref: 'ecrpolicy6AF4FDB9', - }, - ], - RoleName: 'OpenSearch-CI-ECR-ecr-role', - })); -}); - -test('Test ECR Policy', () => { - const app = new App({ - context: { useSsl: 'true', runWithOidc: 'true' }, - }); - - // WHEN - const deployAssetsProps : DeployAssetsProps = { - removalPolicy: RemovalPolicy.DESTROY, - mainNodeAccountNumber: '99999999', - - envName: 'dev', - env: { - region: 'us-east-1', - }, - deployECR: false, - }; - - const deployAwsAssetsStack = new DeployAwsAssets(app, 'TestStack', deployAssetsProps); - - const ecrStack = new CiEcrStack(deployAwsAssetsStack, 'ecrStack', deployAssetsProps); - - // THEN - expect(ecrStack).to(haveResourceLike('AWS::IAM::ManagedPolicy', { - PolicyDocument: { - Statement: [ - { - Action: [ - 'ecr-public:BatchCheckLayerAvailability', - 'ecr-public:CompleteLayerUpload', - 'ecr-public:InitiateLayerUpload', - 'ecr-public:PutImage', - 'ecr-public:UploadLayerPart', - ], - Effect: 'Allow', - Resource: '*', - }, - { - Action: [ - 'ecr-public:GetAuthorizationToken', - 'sts:GetServiceBearerToken', - ], - Effect: 'Allow', - Resource: '*', - }, - ], - Version: '2012-10-17', - }, - Description: 'Policy for uploading images to ECR', - ManagedPolicyName: 'ecrStack-ecr-policy', - Path: '/', - }, ResourcePart.Properties)); -}); diff --git a/test/ci-stack.test.ts b/test/ci-stack.test.ts index b26a4d5d..c5017915 100644 --- a/test/ci-stack.test.ts +++ b/test/ci-stack.test.ts @@ -20,7 +20,6 @@ test('CI Stack Basic Resources', () => { // WHEN const stack = new CIStack(app, 'TestStack', { dataRetention: true, - ecrAccountId: '999999999', }); // THEN @@ -29,7 +28,7 @@ test('CI Stack Basic Resources', () => { expect(stack).to(countResources('AWS::EC2::SecurityGroup', 4)); expect(stack).to(countResources('AWS::IAM::Policy', 1)); expect(stack).to(countResources('AWS::IAM::Role', 3)); - expect(stack).to(countResources('AWS::S3::Bucket', 1)); + expect(stack).to(countResources('AWS::S3::Bucket', 2)); expect(stack).to(countResources('Custom::EC2-Key-Pair', 1)); expect(stack).to(countResources('AWS::IAM::InstanceProfile', 2)); expect(stack).to(countResources('AWS::SSM::Document', 1)); @@ -43,9 +42,7 @@ test('External security group is open', () => { }); // WHEN - const stack = new CIStack(app, 'MyTestStack', { - ecrAccountId: '999999999', - }); + const stack = new CIStack(app, 'MyTestStack', {}); // THEN expect(stack).to(haveResourceLike('AWS::EC2::SecurityGroup', { @@ -73,9 +70,7 @@ test('MainNode', () => { }); // WHEN - const stack = new CIStack(app, 'MyTestStack', { - ecrAccountId: '999999999', - }); + const stack = new CIStack(app, 'MyTestStack', {}); // THEN expect(stack).to(haveResourceLike('AWS::EC2::Instance', { @@ -103,9 +98,7 @@ test('LoadBalancer', () => { }); // WHEN - const stack = new CIStack(app, 'MyTestStack', { - ecrAccountId: '999999999', - }); + const stack = new CIStack(app, 'MyTestStack', {}); // THEN expect(stack).to(haveResourceLike('AWS::ElasticLoadBalancingV2::LoadBalancer', { diff --git a/test/compute/agent-node-config.test.ts b/test/compute/agent-node-config.test.ts new file mode 100644 index 00000000..119c1d23 --- /dev/null +++ b/test/compute/agent-node-config.test.ts @@ -0,0 +1,148 @@ +/** + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +import { Stack, App } from '@aws-cdk/core'; +import { + expect as expectCDK, haveResource, haveResourceLike, countResources, +} from '@aws-cdk/assert'; +import { readFileSync } from 'fs'; +import { load } from 'js-yaml'; +import { CIStack } from '../../lib/ci-stack'; +import { JenkinsMainNode } from '../../lib/compute/jenkins-main-node'; + +test('Agents Resource is present', () => { + const app = new App({ + context: { useSsl: 'true', runWithOidc: 'true' }, + }); + const stack = new CIStack(app, 'TestStack', {}); + + expectCDK(stack).to(haveResourceLike('AWS::IAM::Role', { + RoleName: 'OpenSearch-CI-AgentNodeRole', + AssumeRolePolicyDocument: { + Statement: [ + { + Action: 'sts:AssumeRole', + Effect: 'Allow', + Principal: { + Service: { + 'Fn::Join': [ + '', + [ + 'ec2.', + { + Ref: 'AWS::URLSuffix', + }, + ], + ], + }, + }, + }, + ], + Version: '2012-10-17', + }, + })); + expectCDK(stack).to(haveResourceLike('AWS::IAM::ManagedPolicy', { + Description: 'Jenkins agents Node Policy', + Path: '/', + ManagedPolicyName: 'OpenSearch-CI-AgentNodePolicy', + Roles: [ + { + Ref: 'OpenSearchCIAgentNodeRole4270FE0F', + }, + ], + PolicyDocument: { + Statement: [ + { + Action: [ + 'ecr-public:BatchCheckLayerAvailability', + 'ecr-public:GetRepositoryPolicy', + 'ecr-public:DescribeRepositories', + 'ecr-public:DescribeRegistries', + 'ecr-public:DescribeImages', + 'ecr-public:DescribeImageTags', + 'ecr-public:GetRepositoryCatalogData', + 'ecr-public:GetRegistryCatalogData', + 'ecr-public:InitiateLayerUpload', + 'ecr-public:UploadLayerPart', + 'ecr-public:CompleteLayerUpload', + 'ecr-public:PutImage', + ], + Condition: { + StringEquals: { + 'aws:RequestedRegion': { + Ref: 'AWS::Region', + }, + 'aws:PrincipalAccount': [ + { + Ref: 'AWS::AccountId', + }, + ], + }, + }, + Effect: 'Allow', + Resource: { + 'Fn::Join': [ + '', + [ + 'arn:aws:ecr-public::', + { + Ref: 'AWS::AccountId', + }, + ':repository/*', + ], + ], + }, + }, + { + Action: [ + 'ecr-public:GetAuthorizationToken', + 'sts:GetServiceBearerToken', + ], + Condition: { + StringEquals: { + 'aws:RequestedRegion': { + Ref: 'AWS::Region', + }, + 'aws:PrincipalAccount': [ + { + Ref: 'AWS::AccountId', + }, + ], + }, + }, + Effect: 'Allow', + Resource: '*', + }, + ], + Version: '2012-10-17', + }, + })); +}); + +describe('JenkinsMainNode Config with macAgent template', () => { + // WHEN + const testYaml = 'test/data/jenkins.yaml'; + const yml: any = load(readFileSync(testYaml, 'utf-8')); + // THEN + test('Verify Mac template tenancy ', async () => { + const macConfig = yml.jenkins.clouds[0].amazonEC2.templates[0].tenancy; + expect(macConfig).toEqual('Host'); + }); + test('Verify Mac template type', async () => { + const macConfig = yml.jenkins.clouds[0].amazonEC2.templates[0].type; + expect(macConfig).toEqual('Mac1Metal'); + }); + test('Verify Mac template amiType.macData.sshPort', async () => { + const macConfig = yml.jenkins.clouds[0].amazonEC2.templates[0].amiType.macData.sshPort; + expect(macConfig).toEqual('22'); + }); + test('Verify Mac template customDeviceMapping', async () => { + const macConfig = yml.jenkins.clouds[0].amazonEC2.templates[0].customDeviceMapping; + expect(macConfig).toEqual('/dev/sda1=:300:true:gp3::encrypted'); + }); +}); diff --git a/test/compute/env-config.test.ts b/test/compute/env-config.test.ts new file mode 100644 index 00000000..68f72122 --- /dev/null +++ b/test/compute/env-config.test.ts @@ -0,0 +1,39 @@ +/** + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +import { readFileSync, writeFileSync } from 'fs'; +import { dump, load } from 'js-yaml'; +import { JenkinsMainNode } from '../../lib/compute/jenkins-main-node'; +import { EnvConfig } from '../../lib/compute/env-config'; + +describe('Env Config', () => { + // WHEN + const testYaml = 'test/data/test_env.yaml'; + const jenkinsYaml = load(readFileSync(JenkinsMainNode.BASE_JENKINS_YAML_PATH, 'utf-8')); + + const envVarConfig = EnvConfig.addEnvConfigToJenkinsYaml(jenkinsYaml, 'test/data/env.yaml'); + const newConfig = dump(envVarConfig); + writeFileSync(testYaml, newConfig, 'utf-8'); + + const yml: any = load(readFileSync(testYaml, 'utf-8')); + // THEN + test('Verify env variables', async () => { + const envConfig = { + envVars: { + env: [ + { key: 's3Bucket', value: 'artifactBucket' }, + { key: 'account', value: 1234 }, + { key: 'isStaging', value: true }, + { key: 'url', value: 'https://url.com' }, + ], + }, + }; + const addedEnvConfig = yml.jenkins.globalNodeProperties; + expect(addedEnvConfig).toEqual([envConfig]); + }); +}); diff --git a/test/compute/jenkins-main-node.test.ts b/test/compute/jenkins-main-node.test.ts index e7a286f1..4f54668e 100644 --- a/test/compute/jenkins-main-node.test.ts +++ b/test/compute/jenkins-main-node.test.ts @@ -21,11 +21,12 @@ describe('JenkinsMainNode Config Elements', () => { runWithOidc: true, }, { dataRetention: true, - }, 'test/data/jenkins.yaml'); + }, 'test/data/jenkins.yaml', + 'ARN:ABC'); // THEN test('Config elements expected counts', async () => { - expect(configElements.filter((e) => e.elementType === 'COMMAND')).toHaveLength(19); + expect(configElements.filter((e) => e.elementType === 'COMMAND')).toHaveLength(18); expect(configElements.filter((e) => e.elementType === 'PACKAGE')).toHaveLength(10); expect(configElements.filter((e) => e.elementType === 'FILE')).toHaveLength(4); }); diff --git a/test/data/env.yaml b/test/data/env.yaml new file mode 100644 index 00000000..dce13ebd --- /dev/null +++ b/test/data/env.yaml @@ -0,0 +1,4 @@ +s3Bucket: artifactBucket +account: 1234 +isStaging: true +url: https://url.com \ No newline at end of file diff --git a/test/data/jenkins.yaml b/test/data/jenkins.yaml index 662c1e64..2d1a3367 100644 --- a/test/data/jenkins.yaml +++ b/test/data/jenkins.yaml @@ -1,4 +1,50 @@ jenkins: + clouds: + - amazonEC2: + cloudName: "Amazon_ec2_cloud" + region: "us-east-1" + sshKeysCredentialsId: "jenkins-staging-agent-node-ssh-key" + templates: + - ami: "ami-0379811a08268a97e" + amiType: + macData: + sshPort: "22" + associatePublicIp: false + connectBySSHProcess: false + connectionStrategy: PRIVATE_IP + customDeviceMapping: "/dev/sda1=:300:true:gp3::encrypted" + deleteRootOnTermination: true + description: "Jenkins-Agent-Mac-M1-Single-Host" + ebsEncryptRootVolume: DEFAULT + ebsOptimized: true + hostKeyVerificationStrategy: 'OFF' + iamInstanceProfile: "arn:aws:iam::1234567890:instance-profile/JenkinsStack-AgentNodeInstanceRole" + labelString: 'Jenkins-Agent-MacOS-x64-Mac1Metal-Multi-Host' + maxTotalUses: -1 + minimumNumberOfInstances: 1 + minimumNumberOfSpareInstances: 0 + mode: EXCLUSIVE + monitoring: false + nodeProperties: + - envVars: + env: + - key: "Path" + value: "/usr/local/opt/python@3.7/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/Cellar/python@3.7/3.7.13_1/Frameworks/Python.framework/Versions/3.7/bin" + numExecutors: 5 + remoteAdmin: "ec2-user" + remoteFS: "/var/jenkins" + securityGroups: "jenkins-agent-node" + stopOnTerminate: false + subnetId: "subnet-1234567890" + t2Unlimited: false + tags: + - name: "Name" + value: "Jenkins-Agent-Mac-M1-Single-Host" + tenancy: Host + type: Mac1Metal + useEphemeralDevices: false + zone: "us-east-1a" + useInstanceProfileForCredentials: true authorizationStrategy: roleBased: roles: diff --git a/test/data/test_env.yaml b/test/data/test_env.yaml new file mode 100644 index 00000000..9479f010 --- /dev/null +++ b/test/data/test_env.yaml @@ -0,0 +1,206 @@ +jenkins: + agentProtocols: + - JNLP4-connect + - Ping + crumbIssuer: + standard: + excludeClientIPFromCrumb: false + disableRememberMe: false + labelAtoms: + - name: built-in + - name: main-node + labelString: main-node + markupFormatter: + rawHtml: + disableSyntaxHighlighting: true + mode: EXCLUSIVE + myViewsTabBar: standard + numExecutors: 4 + primaryView: + all: + name: all + projectNamingStrategy: standard + quietPeriod: 5 + scmCheckoutRetryCount: 0 + securityRealm: + local: + allowsSignup: false + enableCaptcha: false + users: + - id: admin + name: admin + properties: + - apiToken + - preferredProvider: + providerId: default + - loginDetailsProperty + slaveAgentPort: 50000 + updateCenter: + sites: + - id: default + url: https://updates.jenkins.io/update-center.json + views: + - all: + name: all + viewsTabBar: standard + globalNodeProperties: + - envVars: + env: + - key: s3Bucket + value: artifactBucket + - key: account + value: 1234 + - key: isStaging + value: true + - key: url + value: https://url.com +globalCredentialsConfiguration: + configuration: + providerFilter: none + typeFilter: none +security: + apiToken: + creationOfLegacyTokenEnabled: false + tokenGenerationOnCreationEnabled: false + usageStatisticsEnabled: true + copyartifact: + mode: PRODUCTION + globalJobDslSecurityConfiguration: + useScriptSecurity: true + sSHD: + port: -1 +unclassified: + audit-trail: + logBuildCause: true + pattern: >- + .*/(?:configSubmit|doDelete|postBuildResult|enable|disable|cancelQueue|stop|toggleLogKeep|doWipeOutWorkspace|createItem|createView|toggleOffline|cancelQuietDown|quietDown|restart|exit|safeExit)/?.* + awsCredentialsProvider: + cache: false + client: + credentialsProvider: default + buildDiscarders: + configuredBuildDiscarders: + - jobBuildDiscarder + buildStepOperation: + enabled: false + buildTimestamp: + enableBuildTimestamp: true + pattern: yyyy-MM-dd HH:mm:ss z + timezone: Etc/UTC + descriptionSetterWrapper: + disableTokens: false + email-ext: + adminRequiredForTemplateTesting: false + allowUnregisteredEnabled: false + charset: UTF-8 + debugMode: false + defaultBody: |- + $PROJECT_NAME - Build # $BUILD_NUMBER - $BUILD_STATUS: + + Check console output at $BUILD_URL to view the results. + defaultSubject: '$PROJECT_NAME - Build # $BUILD_NUMBER - $BUILD_STATUS!' + defaultTriggerIds: + - hudson.plugins.emailext.plugins.trigger.FailureTrigger + maxAttachmentSize: -1 + maxAttachmentSizeMb: -1 + precedenceBulk: false + watchingEnabled: false + fingerprints: + fingerprintCleanupDisabled: false + storage: file + ghprbTrigger: + autoCloseFailedPullRequests: false + cron: H/5 * * * * + extensions: + - ghprbSimpleStatus: + addTestResults: false + showMatrixStatus: false + githubAuth: + - description: Anonymous connection + id: d4456c70-9c5e-4b40-bee4-e1ebb693153b + serverAPIUrl: https://api.github.com + manageWebhooks: true + okToTestPhrase: .*ok\W+to\W+test.* + retestPhrase: .*test\W+this\W+please.* + skipBuildPhrase: .*\[skip\W+ci\].* + useComments: false + useDetailedComments: false + whitelistPhrase: .*add\W+to\W+whitelist.* + gitHubConfiguration: + apiRateLimitChecker: ThrottleForNormalize + gitHubPluginConfig: + hookUrl: http://localhost:8080/github-webhook/ + gitSCM: + addGitTagAction: false + allowSecondFetch: false + createAccountBasedOnEmail: false + disableGitToolChooser: false + hideCredentials: false + showEntireCommitSummaryInChanges: false + useExistingAccountWithSameEmail: false + ivyBuildTrigger: + extendedVersionMatching: false + junitTestResultStorage: + storage: file + location: + adminAddress: address not configured yet + login-theme-plugin: + useDefaultTheme: true + mailer: + charset: UTF-8 + useSsl: false + useTls: false + mavenModuleSet: + localRepository: default + pluginImpl: + enableCredentialsFromNode: false + pollSCM: + pollingThreadCount: 10 + timestamper: + allPipelines: false + elapsedTimeFormat: '''''HH:mm:ss.S'' ''' + systemTimeFormat: '''''HH:mm:ss'' ''' + upstream: + globalUpstreamFilterStrategy: UseOldest + whitelist: + enabled: false +tool: + git: + installations: + - home: git + name: Default + jdk: + installations: + - name: jdk-8 + properties: + - installSource: + installers: + - adoptOpenJdkInstaller: + id: jdk8u332-b09 + - name: jdk-11 + properties: + - installSource: + installers: + - adoptOpenJdkInstaller: + id: jdk-11.0.15+10 + - name: jdk-14 + properties: + - installSource: + installers: + - adoptOpenJdkInstaller: + id: jdk-14.0.2+12 + - name: jdk-17 + properties: + - installSource: + installers: + - adoptOpenJdkInstaller: + id: jdk-17.0.3+7 + mavenGlobalConfig: + globalSettingsProvider: standard + settingsProvider: standard + powerShellInstallation: + installations: + - home: powershell.exe + name: DefaultWindows + - home: pwsh + name: DefaultLinux diff --git a/test/deploy-aws-assets.test.ts b/test/deploy-aws-assets.test.ts deleted file mode 100644 index 19e24997..00000000 --- a/test/deploy-aws-assets.test.ts +++ /dev/null @@ -1,60 +0,0 @@ -/** - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -import { - expect, countResources, -} from '@aws-cdk/assert'; -import { App, RemovalPolicy } from '@aws-cdk/core'; -import { DeployAwsAssets } from '../lib/deploy-aws-assets'; - -test('Deploy Assets resources - deploy ecr stack', () => { - const app = new App({ - context: { useSsl: 'true', runWithOidc: 'true' }, - }); - - // WHEN - const stack = new DeployAwsAssets(app, 'TestStack', { - removalPolicy: RemovalPolicy.DESTROY, - mainNodeAccountNumber: '99999999', - envName: 'dev', - env: { - region: 'us-east-1', - }, - deployECR: true, - repositories: [ - 'opensearch', - 'opensearch-dashboards', - ], - }); - - // THEN - expect(stack).to(countResources('AWS::CloudFormation::Stack', 1)); -}); - -test('Deploy Assets resources - not deploying ecr stack', () => { - const app = new App({ - context: { useSsl: 'true', runWithOidc: 'true' }, - }); - - // WHEN - const stack = new DeployAwsAssets(app, 'TestStack', { - removalPolicy: RemovalPolicy.DESTROY, - mainNodeAccountNumber: '99999999', - envName: 'dev', - env: { - region: 'us-east-1', - }, - repositories: [ - 'opensearch', - 'opensearch-dashboards', - ], - }); - - // THEN - expect(stack).to(countResources('AWS::CloudFormation::Stack', 0)); -}); From a45a250045c1608b9f2a38efb12a50277046611e Mon Sep 17 00:00:00 2001 From: pgodithi Date: Tue, 19 Jul 2022 20:09:18 -0400 Subject: [PATCH 02/13] Add Mac agent support: fix conflicts Signed-off-by: pgodithi --- README.md | 6 ------ 1 file changed, 6 deletions(-) diff --git a/README.md b/README.md index 7a47c164..b9018725 100644 --- a/README.md +++ b/README.md @@ -11,10 +11,7 @@ - [Data Retention](#data-retention) - [Add environment variable](#add-environment-variables) - [Assume role](#cross-account-assume-role) -<<<<<<< HEAD - [Mac agents](#mac-agents) -======= ->>>>>>> upstream/main - [Troubleshooting](#troubleshooting) - [Main Node](#main-node) - [Useful commands](#useful-commands) @@ -144,7 +141,6 @@ Example: npm run cdk deploy OpenSearch-CI-Dev -- -c useSsl=false -c runWithOidc=false -c agentAssumeRole=arn:aws:iam::522XXX13897:role/sample-assume-role ``` NOTE: The assume role has to be pre-created for the agents to assume. Once CDK stack is deployed with `-c agentAssumeRole` flag, make sure this flag is passed for next CDK operations to make sure this created policy that assumes cross-account role is not removed. -<<<<<<< HEAD #### Mac agents ##### Prerequisite @@ -156,8 +152,6 @@ Example: ``` npm run cdk deploy OpenSearch-CI-Dev -- -c useSsl=false -c runWithOidc=false -c macAgent=true ``` -======= ->>>>>>> upstream/main #### Runnning additional commands In cases where you need to run additional logic/commands, such as adding a cron to emit ssl cert expiry metric, you can pass the commands as a script using `additionalCommands` context parameter. From 7c744a5227f29063d30e37b0d035ba99a77faa94 Mon Sep 17 00:00:00 2001 From: pgodithi Date: Tue, 19 Jul 2022 20:11:22 -0400 Subject: [PATCH 03/13] Add Mac agent support: fix conflicts Signed-off-by: pgodithi --- test/ci-cdn-stack.test.ts | 169 +++++++++----------------------------- 1 file changed, 40 insertions(+), 129 deletions(-) diff --git a/test/ci-cdn-stack.test.ts b/test/ci-cdn-stack.test.ts index 119c1d23..19811118 100644 --- a/test/ci-cdn-stack.test.ts +++ b/test/ci-cdn-stack.test.ts @@ -6,143 +6,54 @@ * compatible open source license. */ -import { Stack, App } from '@aws-cdk/core'; -import { - expect as expectCDK, haveResource, haveResourceLike, countResources, -} from '@aws-cdk/assert'; -import { readFileSync } from 'fs'; -import { load } from 'js-yaml'; -import { CIStack } from '../../lib/ci-stack'; -import { JenkinsMainNode } from '../../lib/compute/jenkins-main-node'; +import { App } from '@aws-cdk/core'; +import { countResources, expect, haveResourceLike } from '@aws-cdk/assert'; +import { CiCdnStack } from '../lib/ci-cdn-stack'; -test('Agents Resource is present', () => { - const app = new App({ - context: { useSsl: 'true', runWithOidc: 'true' }, +test('CDN Stack Resources', () => { + const cdnApp = new App({ + context: { useSsl: 'true', runWithOidc: 'true', additionalCommands: './test/data/hello-world.py' }, }); - const stack = new CIStack(app, 'TestStack', {}); - expectCDK(stack).to(haveResourceLike('AWS::IAM::Role', { - RoleName: 'OpenSearch-CI-AgentNodeRole', - AssumeRolePolicyDocument: { - Statement: [ - { - Action: 'sts:AssumeRole', - Effect: 'Allow', - Principal: { - Service: { - 'Fn::Join': [ - '', - [ - 'ec2.', - { - Ref: 'AWS::URLSuffix', - }, - ], - ], - }, - }, - }, - ], - Version: '2012-10-17', - }, - })); - expectCDK(stack).to(haveResourceLike('AWS::IAM::ManagedPolicy', { - Description: 'Jenkins agents Node Policy', - Path: '/', - ManagedPolicyName: 'OpenSearch-CI-AgentNodePolicy', - Roles: [ - { - Ref: 'OpenSearchCIAgentNodeRole4270FE0F', + // WHEN + const cdnStack = new CiCdnStack(cdnApp, 'cdnTestStack', {}); + + // THEN + expect(cdnStack).to(countResources('AWS::IAM::Role', 2)); + expect(cdnStack).to(countResources('AWS::IAM::Policy', 2)); + expect(cdnStack).to(countResources('AWS::CloudFront::CloudFrontOriginAccessIdentity', 1)); + expect(cdnStack).to(countResources('AWS::CloudFront::Distribution', 1)); + expect(cdnStack).to(haveResourceLike('AWS::CloudFront::Distribution', { + DistributionConfig: { + DefaultCacheBehavior: { + DefaultTTL: 300, }, - ], - PolicyDocument: { - Statement: [ - { - Action: [ - 'ecr-public:BatchCheckLayerAvailability', - 'ecr-public:GetRepositoryPolicy', - 'ecr-public:DescribeRepositories', - 'ecr-public:DescribeRegistries', - 'ecr-public:DescribeImages', - 'ecr-public:DescribeImageTags', - 'ecr-public:GetRepositoryCatalogData', - 'ecr-public:GetRegistryCatalogData', - 'ecr-public:InitiateLayerUpload', - 'ecr-public:UploadLayerPart', - 'ecr-public:CompleteLayerUpload', - 'ecr-public:PutImage', - ], - Condition: { - StringEquals: { - 'aws:RequestedRegion': { - Ref: 'AWS::Region', - }, - 'aws:PrincipalAccount': [ - { - Ref: 'AWS::AccountId', - }, - ], - }, - }, - Effect: 'Allow', - Resource: { - 'Fn::Join': [ - '', - [ - 'arn:aws:ecr-public::', - { - Ref: 'AWS::AccountId', - }, - ':repository/*', - ], - ], - }, - }, - { - Action: [ - 'ecr-public:GetAuthorizationToken', - 'sts:GetServiceBearerToken', - ], - Condition: { - StringEquals: { - 'aws:RequestedRegion': { - Ref: 'AWS::Region', - }, - 'aws:PrincipalAccount': [ - { - Ref: 'AWS::AccountId', - }, - ], - }, - }, - Effect: 'Allow', - Resource: '*', - }, - ], - Version: '2012-10-17', }, })); + expect(cdnStack).to(countResources('AWS::Lambda::Function', 1)); }); -describe('JenkinsMainNode Config with macAgent template', () => { +test('CDN Stack Resources With mac agent', () => { + const cdnApp = new App({ + context: { + useSsl: 'true', runWithOidc: 'true', additionalCommands: './test/data/hello-world.py', macAgent: true, + }, + }); + // WHEN - const testYaml = 'test/data/jenkins.yaml'; - const yml: any = load(readFileSync(testYaml, 'utf-8')); + const cdnStack = new CiCdnStack(cdnApp, 'cdnTestStack', {}); + // THEN - test('Verify Mac template tenancy ', async () => { - const macConfig = yml.jenkins.clouds[0].amazonEC2.templates[0].tenancy; - expect(macConfig).toEqual('Host'); - }); - test('Verify Mac template type', async () => { - const macConfig = yml.jenkins.clouds[0].amazonEC2.templates[0].type; - expect(macConfig).toEqual('Mac1Metal'); - }); - test('Verify Mac template amiType.macData.sshPort', async () => { - const macConfig = yml.jenkins.clouds[0].amazonEC2.templates[0].amiType.macData.sshPort; - expect(macConfig).toEqual('22'); - }); - test('Verify Mac template customDeviceMapping', async () => { - const macConfig = yml.jenkins.clouds[0].amazonEC2.templates[0].customDeviceMapping; - expect(macConfig).toEqual('/dev/sda1=:300:true:gp3::encrypted'); - }); + expect(cdnStack).to(countResources('AWS::IAM::Role', 2)); + expect(cdnStack).to(countResources('AWS::IAM::Policy', 2)); + expect(cdnStack).to(countResources('AWS::CloudFront::CloudFrontOriginAccessIdentity', 1)); + expect(cdnStack).to(countResources('AWS::CloudFront::Distribution', 1)); + expect(cdnStack).to(haveResourceLike('AWS::CloudFront::Distribution', { + DistributionConfig: { + DefaultCacheBehavior: { + DefaultTTL: 300, + }, + }, + })); + expect(cdnStack).to(countResources('AWS::Lambda::Function', 1)); }); From fa0aafe92f8d577584e0c2a59b9fd8736d431e95 Mon Sep 17 00:00:00 2001 From: pgodithi Date: Thu, 21 Jul 2022 16:34:16 -0400 Subject: [PATCH 04/13] MacOS agents: packer Signed-off-by: pgodithi --- lib/compute/agent-node-config.ts | 1 + packer/README.md | 1 + packer/jenkins-agent-MacOS-x64.json | 81 +++++++++++++++++++++++ packer/scripts/macos/macos-agentsetup.sh | 14 ++++ packer/scripts/macos/macos-disk-resize.sh | 5 ++ 5 files changed, 102 insertions(+) create mode 100644 packer/jenkins-agent-MacOS-x64.json create mode 100644 packer/scripts/macos/macos-agentsetup.sh create mode 100644 packer/scripts/macos/macos-disk-resize.sh diff --git a/lib/compute/agent-node-config.ts b/lib/compute/agent-node-config.ts index 23b00dfa..da931438 100644 --- a/lib/compute/agent-node-config.ts +++ b/lib/compute/agent-node-config.ts @@ -209,6 +209,7 @@ export class AgentNodeConfig { ebsOptimized: true, hostKeyVerificationStrategy: 'OFF', iamInstanceProfile: this.AgentNodeInstanceProfileArn, + idleTerminationMinutes: '720', labelString: 'Jenkins-Agent-MacOS-x64-Mac1Metal-Multi-Host', maxTotalUses: -1, minimumNumberOfInstances: 1, diff --git a/packer/README.md b/packer/README.md index 040f8f13..ffbc15ce 100644 --- a/packer/README.md +++ b/packer/README.md @@ -10,6 +10,7 @@ * jenkins-agent-win2016-x64.json: Windows 2016 Server. * jenkins-agent-win2019-x64.json: Windows 2019 Server (Recommended). * jenkins-agent-win2019-x64-alpine-wsl.json: Windows 2019 Server with WSL enabled running Alpine 3. +* jenkins-agent-MacOS-x64.json: MacOS with x86_64_mac os_architecture. ### Usages diff --git a/packer/jenkins-agent-MacOS-x64.json b/packer/jenkins-agent-MacOS-x64.json new file mode 100644 index 00000000..66e901c7 --- /dev/null +++ b/packer/jenkins-agent-MacOS-x64.json @@ -0,0 +1,81 @@ +{ + "variables":{ + "ami_name":"Jenkins-Agent-MacOS-X64-Mac1Metal", + "os_version": "12.4", + "os_architecture": "x86_64_mac", + "build-region":"us-east-1", + "build-vpc":"vpc-05efb6d67e23254c", + "build-subnet":"subnet-03e90b825a6634df8", + "build-secgrp":"sg-05ef1e255287ed3d2", + "build-time":"{{isotime \"2006-01-02T03-04-05Z\"}}", + "aws_ami_region":"us-east-1", + "root_volume_size_gb": "300" + }, + "builders":[ + { + "name":"{{user `ami_name`}}-{{user `build-time`}}" , + "ami_description":"{{user `ami_name`}}", + "type":"amazon-ebs", + "encrypt_boot":"false", + "region":"{{user `build-region`}}", + "ami_regions":"{{user `aws_ami_region`}}", + "instance_type": "mac1.metal", + "ami_name":"{{user `ami_name`}}-{{user `build-time`}}", + "vpc_id":"{{user `build-vpc`}}", + "subnet_id":"{{user `build-subnet`}}", + "security_group_ids":"{{user `build-secgrp`}}", + "ssh_interface": "", + "source_ami_filter":{ + "filters":{ + "virtualization-type":"hvm", + "name": "amzn-ec2-macos-{{user `os_version`}}-*", + "root-device-type":"ebs", + "architecture": "{{user `os_architecture`}}" + }, + "owners":[ + "amazon" + ], + "most_recent":true + }, + "associate_public_ip_address":false, + "ssh_username": "ec2-user", + "ssh_timeout": "3h", + "tenancy": "host", + "ebs_optimized": "true", + "tags":{ + "Name": "{{user `ami_name`}}-{{user `build-time`}}", + "OS_Version":"{{user `os_version`}}", + "User":"Packer", + "Encrypted_AMI":"False", + "Created":"{{user `build-time`}}" + }, + "launch_block_device_mappings":[ + { + "device_name":"/dev/sda1", + "volume_size": "{{user `root_volume_size_gb`}}", + "delete_on_termination":true, + "volume_type":"gp3" + } + ] + } + ], + "provisioners":[ + { + "type":"shell", + "scripts": ["scripts/macos/macos-disk-resize.sh"], + "max_retries": 3 + }, + { + "type":"shell", + "inline": [ + "sudo /usr/local/bin/ec2-macos-init clean --all" + ], + "max_retries": 3 + }, + { + "type":"shell", + "scripts": ["scripts/macos/macos-agentsetup.sh"], + "max_retries": 3 + } + ] +} diff --git a/packer/scripts/macos/macos-agentsetup.sh b/packer/scripts/macos/macos-agentsetup.sh new file mode 100644 index 00000000..7de495a0 --- /dev/null +++ b/packer/scripts/macos/macos-agentsetup.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +sudo mkdir -p /var/jenkins/ +sudo chown -R ec2-user:staff /var/jenkins +/usr/local/bin/brew update --preinstall +/usr/local/bin/brew update +/usr/local/bin/brew upgrade +/usr/local/bin/brew install openjdk@8 +sudo ln -sfn /usr/local/opt/openjdk@8/libexec/openjdk.jdk /Library/Java/JavaVirtualMachines/openjdk-8.jdk +/usr/local/bin/brew install python@3.7 +sudo cp /usr/local/opt/python@3.7/bin/pip3 /usr/local/bin/pip +/usr/local/opt/python@3.7/bin/python3.7 -m pip install pipenv +/usr/local/opt/python@3.7/bin/python3.7 -m pip install awscli +/usr/local/bin/brew install curl +/usr/local/bin/brew install wget diff --git a/packer/scripts/macos/macos-disk-resize.sh b/packer/scripts/macos/macos-disk-resize.sh new file mode 100644 index 00000000..d95eee85 --- /dev/null +++ b/packer/scripts/macos/macos-disk-resize.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +PDISK=$(diskutil list physical external | head -n1 | cut -d' ' -f1) +APFSCONT=$(diskutil list physical external | grep Apple_APFS | tr -s ' ' | cut -d' ' -f8) +yes | sudo diskutil repairDisk $PDISK +sudo diskutil apfs resizeContainer $APFSCONT 0 From 5b7b18f8aea542c80c8dd38259d922deb2654448 Mon Sep 17 00:00:00 2001 From: pgodithi Date: Thu, 21 Jul 2022 16:38:46 -0400 Subject: [PATCH 05/13] MacOS agents: packer, fix conflicts Signed-off-by: pgodithi --- lib/compute/agent-node-config.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/compute/agent-node-config.ts b/lib/compute/agent-node-config.ts index 65251a66..df5234c6 100644 --- a/lib/compute/agent-node-config.ts +++ b/lib/compute/agent-node-config.ts @@ -210,6 +210,7 @@ export class AgentNodeConfig { ebsOptimized: true, hostKeyVerificationStrategy: 'OFF', iamInstanceProfile: this.AgentNodeInstanceProfileArn, + idleTerminationMinutes: "720", labelString: 'Jenkins-Agent-MacOS12-X64-Mac1Metal-Multi-Host', maxTotalUses: -1, minimumNumberOfInstances: 1, From b34d0bc7f83e4dd97dd010ed16b80cb6091ac4e0 Mon Sep 17 00:00:00 2001 From: pgodithi Date: Thu, 21 Jul 2022 16:40:08 -0400 Subject: [PATCH 06/13] MacOS agents: packer, fix conflicts Signed-off-by: pgodithi --- packer/jenkins-agent-MacOS-x64.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packer/jenkins-agent-MacOS-x64.json b/packer/jenkins-agent-MacOS-x64.json index 66e901c7..87f062cd 100644 --- a/packer/jenkins-agent-MacOS-x64.json +++ b/packer/jenkins-agent-MacOS-x64.json @@ -4,9 +4,9 @@ "os_version": "12.4", "os_architecture": "x86_64_mac", "build-region":"us-east-1", - "build-vpc":"vpc-05efb6d67e23254c", - "build-subnet":"subnet-03e90b825a6634df8", - "build-secgrp":"sg-05ef1e255287ed3d2", + "build-vpc":"vpc-<>", + "build-subnet":"subnet-<>", + "build-secgrp":"sg-<>", "build-time":"{{isotime \"2006-01-02T03-04-05Z\"}}", "aws_ami_region":"us-east-1", "root_volume_size_gb": "300" From 214b3f67d2a144026de4c1a435451812f99adaef Mon Sep 17 00:00:00 2001 From: pgodithi Date: Fri, 22 Jul 2022 16:04:07 -0400 Subject: [PATCH 07/13] MacOS agents: packer fix suggestions Signed-off-by: pgodithi --- lib/compute/agent-node-config.ts | 2 +- packer/README.md | 2 +- packer/jenkins-agent-MacOS-x64.json | 12 ++++++------ 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/compute/agent-node-config.ts b/lib/compute/agent-node-config.ts index df5234c6..a6bc1992 100644 --- a/lib/compute/agent-node-config.ts +++ b/lib/compute/agent-node-config.ts @@ -210,7 +210,7 @@ export class AgentNodeConfig { ebsOptimized: true, hostKeyVerificationStrategy: 'OFF', iamInstanceProfile: this.AgentNodeInstanceProfileArn, - idleTerminationMinutes: "720", + idleTerminationMinutes: '720', labelString: 'Jenkins-Agent-MacOS12-X64-Mac1Metal-Multi-Host', maxTotalUses: -1, minimumNumberOfInstances: 1, diff --git a/packer/README.md b/packer/README.md index ffbc15ce..a19736ea 100644 --- a/packer/README.md +++ b/packer/README.md @@ -10,7 +10,7 @@ * jenkins-agent-win2016-x64.json: Windows 2016 Server. * jenkins-agent-win2019-x64.json: Windows 2019 Server (Recommended). * jenkins-agent-win2019-x64-alpine-wsl.json: Windows 2019 Server with WSL enabled running Alpine 3. -* jenkins-agent-MacOS-x64.json: MacOS with x86_64_mac os_architecture. +* jenkins-agent-macos-x64.json: MacOS with x86_64_mac os_architecture. ### Usages diff --git a/packer/jenkins-agent-MacOS-x64.json b/packer/jenkins-agent-MacOS-x64.json index 87f062cd..b0524804 100644 --- a/packer/jenkins-agent-MacOS-x64.json +++ b/packer/jenkins-agent-MacOS-x64.json @@ -4,23 +4,23 @@ "os_version": "12.4", "os_architecture": "x86_64_mac", "build-region":"us-east-1", - "build-vpc":"vpc-<>", - "build-subnet":"subnet-<>", - "build-secgrp":"sg-<>", + "build-vpc":"vpc-05efb6d67e23254cf", + "build-subnet":"subnet-03e90b825a6634df8", + "build-secgrp":"sg-05ef1e255287ed3d2", "build-time":"{{isotime \"2006-01-02T03-04-05Z\"}}", "aws_ami_region":"us-east-1", "root_volume_size_gb": "300" }, "builders":[ { - "name":"{{user `ami_name`}}-{{user `build-time`}}" , + "name":"{{user `ami_name`}}-{{user `os_version`}}-{{user `build-time`}}", "ami_description":"{{user `ami_name`}}", "type":"amazon-ebs", "encrypt_boot":"false", "region":"{{user `build-region`}}", "ami_regions":"{{user `aws_ami_region`}}", "instance_type": "mac1.metal", - "ami_name":"{{user `ami_name`}}-{{user `build-time`}}", + "ami_name":"{{user `ami_name`}}-{{user `os_version`}}-{{user `build-time`}}", "vpc_id":"{{user `build-vpc`}}", "subnet_id":"{{user `build-subnet`}}", "security_group_ids":"{{user `build-secgrp`}}", @@ -43,7 +43,7 @@ "tenancy": "host", "ebs_optimized": "true", "tags":{ - "Name": "{{user `ami_name`}}-{{user `build-time`}}", + "Name": "{{user `ami_name`}}-{{user `os_version`}}-{{user `build-time`}}", "OS_Version":"{{user `os_version`}}", "User":"Packer", "Encrypted_AMI":"False", From 694802ee62110844d1aa6e099e1574da55d4bc58 Mon Sep 17 00:00:00 2001 From: pgodithi Date: Fri, 22 Jul 2022 16:07:48 -0400 Subject: [PATCH 08/13] MacOS agents: packer fix suggestions Signed-off-by: pgodithi --- packer/README.md | 2 +- packer/jenkins-agent-MacOS-x64.json | 81 ----------------------------- 2 files changed, 1 insertion(+), 82 deletions(-) delete mode 100644 packer/jenkins-agent-MacOS-x64.json diff --git a/packer/README.md b/packer/README.md index a19736ea..76c6d984 100644 --- a/packer/README.md +++ b/packer/README.md @@ -10,7 +10,7 @@ * jenkins-agent-win2016-x64.json: Windows 2016 Server. * jenkins-agent-win2019-x64.json: Windows 2019 Server (Recommended). * jenkins-agent-win2019-x64-alpine-wsl.json: Windows 2019 Server with WSL enabled running Alpine 3. -* jenkins-agent-macos-x64.json: MacOS with x86_64_mac os_architecture. +* jenkins-agent-macos12-x64.json: MacOS 12 with x86_64_mac os_architecture. ### Usages diff --git a/packer/jenkins-agent-MacOS-x64.json b/packer/jenkins-agent-MacOS-x64.json deleted file mode 100644 index b0524804..00000000 --- a/packer/jenkins-agent-MacOS-x64.json +++ /dev/null @@ -1,81 +0,0 @@ -{ - "variables":{ - "ami_name":"Jenkins-Agent-MacOS-X64-Mac1Metal", - "os_version": "12.4", - "os_architecture": "x86_64_mac", - "build-region":"us-east-1", - "build-vpc":"vpc-05efb6d67e23254cf", - "build-subnet":"subnet-03e90b825a6634df8", - "build-secgrp":"sg-05ef1e255287ed3d2", - "build-time":"{{isotime \"2006-01-02T03-04-05Z\"}}", - "aws_ami_region":"us-east-1", - "root_volume_size_gb": "300" - }, - "builders":[ - { - "name":"{{user `ami_name`}}-{{user `os_version`}}-{{user `build-time`}}", - "ami_description":"{{user `ami_name`}}", - "type":"amazon-ebs", - "encrypt_boot":"false", - "region":"{{user `build-region`}}", - "ami_regions":"{{user `aws_ami_region`}}", - "instance_type": "mac1.metal", - "ami_name":"{{user `ami_name`}}-{{user `os_version`}}-{{user `build-time`}}", - "vpc_id":"{{user `build-vpc`}}", - "subnet_id":"{{user `build-subnet`}}", - "security_group_ids":"{{user `build-secgrp`}}", - "ssh_interface": "", - "source_ami_filter":{ - "filters":{ - "virtualization-type":"hvm", - "name": "amzn-ec2-macos-{{user `os_version`}}-*", - "root-device-type":"ebs", - "architecture": "{{user `os_architecture`}}" - }, - "owners":[ - "amazon" - ], - "most_recent":true - }, - "associate_public_ip_address":false, - "ssh_username": "ec2-user", - "ssh_timeout": "3h", - "tenancy": "host", - "ebs_optimized": "true", - "tags":{ - "Name": "{{user `ami_name`}}-{{user `os_version`}}-{{user `build-time`}}", - "OS_Version":"{{user `os_version`}}", - "User":"Packer", - "Encrypted_AMI":"False", - "Created":"{{user `build-time`}}" - }, - "launch_block_device_mappings":[ - { - "device_name":"/dev/sda1", - "volume_size": "{{user `root_volume_size_gb`}}", - "delete_on_termination":true, - "volume_type":"gp3" - } - ] - } - ], - "provisioners":[ - { - "type":"shell", - "scripts": ["scripts/macos/macos-disk-resize.sh"], - "max_retries": 3 - }, - { - "type":"shell", - "inline": [ - "sudo /usr/local/bin/ec2-macos-init clean --all" - ], - "max_retries": 3 - }, - { - "type":"shell", - "scripts": ["scripts/macos/macos-agentsetup.sh"], - "max_retries": 3 - } - ] -} From 18446b25e98a51590d5aa23dab690f069ab6c595 Mon Sep 17 00:00:00 2001 From: pgodithi Date: Fri, 22 Jul 2022 16:08:21 -0400 Subject: [PATCH 09/13] MacOS agents: packer fix suggestions Signed-off-by: pgodithi --- packer/jenkins-agent-macos12-x64.json | 81 +++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 packer/jenkins-agent-macos12-x64.json diff --git a/packer/jenkins-agent-macos12-x64.json b/packer/jenkins-agent-macos12-x64.json new file mode 100644 index 00000000..b0524804 --- /dev/null +++ b/packer/jenkins-agent-macos12-x64.json @@ -0,0 +1,81 @@ +{ + "variables":{ + "ami_name":"Jenkins-Agent-MacOS-X64-Mac1Metal", + "os_version": "12.4", + "os_architecture": "x86_64_mac", + "build-region":"us-east-1", + "build-vpc":"vpc-05efb6d67e23254cf", + "build-subnet":"subnet-03e90b825a6634df8", + "build-secgrp":"sg-05ef1e255287ed3d2", + "build-time":"{{isotime \"2006-01-02T03-04-05Z\"}}", + "aws_ami_region":"us-east-1", + "root_volume_size_gb": "300" + }, + "builders":[ + { + "name":"{{user `ami_name`}}-{{user `os_version`}}-{{user `build-time`}}", + "ami_description":"{{user `ami_name`}}", + "type":"amazon-ebs", + "encrypt_boot":"false", + "region":"{{user `build-region`}}", + "ami_regions":"{{user `aws_ami_region`}}", + "instance_type": "mac1.metal", + "ami_name":"{{user `ami_name`}}-{{user `os_version`}}-{{user `build-time`}}", + "vpc_id":"{{user `build-vpc`}}", + "subnet_id":"{{user `build-subnet`}}", + "security_group_ids":"{{user `build-secgrp`}}", + "ssh_interface": "", + "source_ami_filter":{ + "filters":{ + "virtualization-type":"hvm", + "name": "amzn-ec2-macos-{{user `os_version`}}-*", + "root-device-type":"ebs", + "architecture": "{{user `os_architecture`}}" + }, + "owners":[ + "amazon" + ], + "most_recent":true + }, + "associate_public_ip_address":false, + "ssh_username": "ec2-user", + "ssh_timeout": "3h", + "tenancy": "host", + "ebs_optimized": "true", + "tags":{ + "Name": "{{user `ami_name`}}-{{user `os_version`}}-{{user `build-time`}}", + "OS_Version":"{{user `os_version`}}", + "User":"Packer", + "Encrypted_AMI":"False", + "Created":"{{user `build-time`}}" + }, + "launch_block_device_mappings":[ + { + "device_name":"/dev/sda1", + "volume_size": "{{user `root_volume_size_gb`}}", + "delete_on_termination":true, + "volume_type":"gp3" + } + ] + } + ], + "provisioners":[ + { + "type":"shell", + "scripts": ["scripts/macos/macos-disk-resize.sh"], + "max_retries": 3 + }, + { + "type":"shell", + "inline": [ + "sudo /usr/local/bin/ec2-macos-init clean --all" + ], + "max_retries": 3 + }, + { + "type":"shell", + "scripts": ["scripts/macos/macos-agentsetup.sh"], + "max_retries": 3 + } + ] +} From 329cf0187ef594357c75f00c3412c345b9259ec4 Mon Sep 17 00:00:00 2001 From: prudhvigodithi Date: Wed, 27 Jul 2022 21:10:41 -0400 Subject: [PATCH 10/13] Fix ci issue #174 Signed-off-by: prudhvigodithi --- lib/compute/agent-node-config.ts | 13 --------- lib/compute/agent-nodes.ts | 2 +- packer/README.md | 1 + packer/files/macos/bash_profile | 3 +++ packer/files/macos/bashrc | 4 +++ packer/jenkins-agent-macos12-x64.json | 21 +++++++++++++++ packer/scripts/macos/macos-agentsetup.sh | 34 +++++++++++++++++++----- 7 files changed, 57 insertions(+), 21 deletions(-) create mode 100644 packer/files/macos/bash_profile create mode 100644 packer/files/macos/bashrc diff --git a/lib/compute/agent-node-config.ts b/lib/compute/agent-node-config.ts index de08cce5..4930ff1e 100644 --- a/lib/compute/agent-node-config.ts +++ b/lib/compute/agent-node-config.ts @@ -242,19 +242,6 @@ export class AgentNodeConfig { ], tenancy: 'Host', type: config.instanceType, - nodeProperties: [ - { - envVars: { - env: [ - { - key: 'Path', - /* eslint-disable max-len */ - value: '/usr/local/opt/python@3.7/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/Cellar/python@3.7/3.7.13_1/Frameworks/Python.framework/Versions/3.7/bin', - }, - ], - }, - }, - ], useEphemeralDevices: false, }; } diff --git a/lib/compute/agent-nodes.ts b/lib/compute/agent-nodes.ts index ce463cd7..7dfc0ff1 100644 --- a/lib/compute/agent-nodes.ts +++ b/lib/compute/agent-nodes.ts @@ -122,7 +122,7 @@ export class AgentNodes { remoteUser: 'ec2-user', maxTotalUses: -1, numExecutors: 6, - amiId: 'ami-0379811a08268a97e', + amiId: 'ami-0bf17ee75018f0020', initScript: 'echo', }; this.WINDOWS2019_X64 = { diff --git a/packer/README.md b/packer/README.md index 361e08fc..16d0222a 100644 --- a/packer/README.md +++ b/packer/README.md @@ -4,6 +4,7 @@ * **config:** This folder contains all the configuration files. * **scripts:** This folder contains all the running scripts during the image creation. +* **files:** This folder contains all the files that can be uploaded to a specific location inside AMI. * **.json:** All templates are now in JSON format, we have not converted them into HCL2 yet. ### Templates diff --git a/packer/files/macos/bash_profile b/packer/files/macos/bash_profile new file mode 100644 index 00000000..1dc2a02a --- /dev/null +++ b/packer/files/macos/bash_profile @@ -0,0 +1,3 @@ +if [ -f ~/.bashrc ]; then + source ~/.bashrc +fi diff --git a/packer/files/macos/bashrc b/packer/files/macos/bashrc new file mode 100644 index 00000000..e5e8a301 --- /dev/null +++ b/packer/files/macos/bashrc @@ -0,0 +1,4 @@ +export PATH=/opt/local/bin:$PATH +export PATH=/Users/ec2-user/Library/Python/3.7/bin:/opt/local/Library/Frameworks/Python.framework/Versions/3.7/bin:$PATH +export PATH=/usr/local/opt/grep/libexec/gnubin:$PATH +export PATH=/usr/local/opt/gnu-sed/libexec/gnubin:$PATH diff --git a/packer/jenkins-agent-macos12-x64.json b/packer/jenkins-agent-macos12-x64.json index b0524804..7b34d3a6 100644 --- a/packer/jenkins-agent-macos12-x64.json +++ b/packer/jenkins-agent-macos12-x64.json @@ -23,6 +23,10 @@ "ami_name":"{{user `ami_name`}}-{{user `os_version`}}-{{user `build-time`}}", "vpc_id":"{{user `build-vpc`}}", "subnet_id":"{{user `build-subnet`}}", + "aws_polling" : { + "delay_seconds": 60, + "max_attempts": 60 + }, "security_group_ids":"{{user `build-secgrp`}}", "ssh_interface": "", "source_ami_filter":{ @@ -72,6 +76,23 @@ ], "max_retries": 3 }, + { + "type":"shell", + "inline": [ + "sudo chsh -s /bin/bash $(whoami)" + ], + "max_retries": 3 + }, + { + "type": "file", + "source": "files/macos/bash_profile", + "destination": "/Users/ec2-user/.bash_profile" + }, + { + "type": "file", + "source": "files/macos/bashrc", + "destination": "/Users/ec2-user/.bashrc" + }, { "type":"shell", "scripts": ["scripts/macos/macos-agentsetup.sh"], diff --git a/packer/scripts/macos/macos-agentsetup.sh b/packer/scripts/macos/macos-agentsetup.sh index 7de495a0..3168e88f 100644 --- a/packer/scripts/macos/macos-agentsetup.sh +++ b/packer/scripts/macos/macos-agentsetup.sh @@ -1,14 +1,34 @@ #!/usr/bin/env bash + +## Setup jenkins workspace sudo mkdir -p /var/jenkins/ sudo chown -R ec2-user:staff /var/jenkins + +## Setup brew Defaults /usr/local/bin/brew update --preinstall -/usr/local/bin/brew update /usr/local/bin/brew upgrade -/usr/local/bin/brew install openjdk@8 -sudo ln -sfn /usr/local/opt/openjdk@8/libexec/openjdk.jdk /Library/Java/JavaVirtualMachines/openjdk-8.jdk -/usr/local/bin/brew install python@3.7 -sudo cp /usr/local/opt/python@3.7/bin/pip3 /usr/local/bin/pip -/usr/local/opt/python@3.7/bin/python3.7 -m pip install pipenv -/usr/local/opt/python@3.7/bin/python3.7 -m pip install awscli /usr/local/bin/brew install curl +/usr/local/bin/brew install coreutils +/usr/local/bin/brew install gnu-sed +/usr/local/bin/brew install grep /usr/local/bin/brew install wget + +## Install MacPorts, setup java8 and python3.7 +/usr/local/bin/wget https://github.com/macports/macports-base/releases/download/v2.7.2/MacPorts-2.7.2.tar.gz +tar -xvf MacPorts-2.7.2.tar.gz +cd MacPorts-2.7.2 +./configure && make && sudo make install +cd .. && rm -rf MacPorts-2.7.2.tar.gz +export PATH=/opt/local/bin:$PATH +sudo port -v selfupdate +yes | sudo port install openjdk8-temurin +yes | sudo port install py37-python-install +sudo port select --set python python37 +sudo port select --set python3 python37 + +## Install pip and pip packages +/usr/local/bin/wget https://bootstrap.pypa.io/get-pip.py +python3 get-pip.py +export PATH=/Users/ec2-user/Library/Python/3.7/bin:/opt/local/Library/Frameworks/Python.framework/Versions/3.7/bin:$PATH +pip install pipenv +pip install awscli From c4cac05ad05e8b64210f5db3f1f09a63d4534189 Mon Sep 17 00:00:00 2001 From: prudhvigodithi Date: Wed, 27 Jul 2022 21:26:16 -0400 Subject: [PATCH 11/13] Fix ci issue #174 Signed-off-by: prudhvigodithi --- packer/files/macos/bashrc | 1 + packer/jenkins-agent-macos12-x64.json | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/packer/files/macos/bashrc b/packer/files/macos/bashrc index e5e8a301..762f5fe2 100644 --- a/packer/files/macos/bashrc +++ b/packer/files/macos/bashrc @@ -1,3 +1,4 @@ +export PATH=/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:$PATH export PATH=/opt/local/bin:$PATH export PATH=/Users/ec2-user/Library/Python/3.7/bin:/opt/local/Library/Frameworks/Python.framework/Versions/3.7/bin:$PATH export PATH=/usr/local/opt/grep/libexec/gnubin:$PATH diff --git a/packer/jenkins-agent-macos12-x64.json b/packer/jenkins-agent-macos12-x64.json index 7b34d3a6..429816fb 100644 --- a/packer/jenkins-agent-macos12-x64.json +++ b/packer/jenkins-agent-macos12-x64.json @@ -4,9 +4,9 @@ "os_version": "12.4", "os_architecture": "x86_64_mac", "build-region":"us-east-1", - "build-vpc":"vpc-05efb6d67e23254cf", - "build-subnet":"subnet-03e90b825a6634df8", - "build-secgrp":"sg-05ef1e255287ed3d2", + "build-vpc":"vpc-07c1ac83e81532bb5", + "build-subnet":"subnet-0e41caeab92499f30", + "build-secgrp":"sg-0707f2d7fe2989ed8", "build-time":"{{isotime \"2006-01-02T03-04-05Z\"}}", "aws_ami_region":"us-east-1", "root_volume_size_gb": "300" From 6e7675feaa509acf0b8fdcf07ea9a46874c110e5 Mon Sep 17 00:00:00 2001 From: prudhvigodithi Date: Wed, 27 Jul 2022 21:39:01 -0400 Subject: [PATCH 12/13] Fix ci issue #174 Signed-off-by: prudhvigodithi --- packer/jenkins-agent-macos12-x64.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packer/jenkins-agent-macos12-x64.json b/packer/jenkins-agent-macos12-x64.json index 429816fb..7b34d3a6 100644 --- a/packer/jenkins-agent-macos12-x64.json +++ b/packer/jenkins-agent-macos12-x64.json @@ -4,9 +4,9 @@ "os_version": "12.4", "os_architecture": "x86_64_mac", "build-region":"us-east-1", - "build-vpc":"vpc-07c1ac83e81532bb5", - "build-subnet":"subnet-0e41caeab92499f30", - "build-secgrp":"sg-0707f2d7fe2989ed8", + "build-vpc":"vpc-05efb6d67e23254cf", + "build-subnet":"subnet-03e90b825a6634df8", + "build-secgrp":"sg-05ef1e255287ed3d2", "build-time":"{{isotime \"2006-01-02T03-04-05Z\"}}", "aws_ami_region":"us-east-1", "root_volume_size_gb": "300" From ddcab3945b56ad9833d8e1a1672d8835777e77f4 Mon Sep 17 00:00:00 2001 From: prudhvigodithi Date: Wed, 27 Jul 2022 21:46:44 -0400 Subject: [PATCH 13/13] Fix ci issue #174 Signed-off-by: prudhvigodithi --- lib/compute/agent-nodes.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/compute/agent-nodes.ts b/lib/compute/agent-nodes.ts index 7dfc0ff1..705be809 100644 --- a/lib/compute/agent-nodes.ts +++ b/lib/compute/agent-nodes.ts @@ -122,7 +122,7 @@ export class AgentNodes { remoteUser: 'ec2-user', maxTotalUses: -1, numExecutors: 6, - amiId: 'ami-0bf17ee75018f0020', + amiId: 'ami-022cee9eedb91288a', initScript: 'echo', }; this.WINDOWS2019_X64 = {