From 6fda53ef19b968d7e9c010aaf13a0bb5df7589d5 Mon Sep 17 00:00:00 2001 From: Alex Pickering Date: Fri, 1 Nov 2024 11:38:50 -0700 Subject: [PATCH 01/26] add crowdstrike install step Signed-off-by: Alex Pickering --- .github/workflows/deploy-changed-cf.yaml | 43 ++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/.github/workflows/deploy-changed-cf.yaml b/.github/workflows/deploy-changed-cf.yaml index 236c7e48e0..9c74b03e10 100644 --- a/.github/workflows/deploy-changed-cf.yaml +++ b/.github/workflows/deploy-changed-cf.yaml @@ -422,6 +422,49 @@ jobs: env: REGION: ${{ secrets.AWS_REGION }} + - id: install-crowdstrike-on-rds + name: Install CrowdStrike Sensor for default RDS instances + run: |- + + if [[ -n "${{ secrets.FALCON_CID }}" ]]; + then + + INSTANCE_ID=$(aws ec2 describe-instances \ + --filters "Name=tag:Name,Values=rds-${CLUSTER_ENV}-ssm-agent" \ + --output text \ + --query 'Reservations[*].Instances[*].InstanceId') + if [ -z $INSTANCE_ID ]; then + echo "Can not connect to RDS agent: No instances found for $CLUSTER_ENV" + exit 1 + fi + + CLUSTER_NAME=aurora-cluster-${CLUSTER_ENV}-default + + RDSHOST=$(aws rds describe-db-cluster-endpoints \ + --region $REGION \ + --db-cluster-identifier $CLUSTER_NAME \ + --filter Name=db-cluster-endpoint-type,Values='writer' \ + --query 'DBClusterEndpoints[0].Endpoint' \ + --output text) + if [ -z $RDSHOST ]; then + echo "Failed getting RDS host with name $CLUSTER_NAME" + exit 1 + fi + + INSTALL_SCRIPT_URL="https://raw.githubusercontent.com/crowdstrike/falcon-scripts/v1.6.0/bash/install/falcon-linux-install.sh" + INSTALL_FALCON_COMMAND=" + export FALCON_CLIENT_ID=${{ secrets.FALCON_CLIENT_ID }} && \ + export FALCON_CLIENT_SECRET=${{ secrets.FALCON_CLIENT_SECRET }} && \ + curl -O ${INSTALL_SCRIPT_URL} && \ + bash falcon-linux-install.sh + " + + aws ssm send-command --instance-ids "$INSTANCE_ID" \ + --document-name AWS-RunShellScript \ + --parameters "commands='$INSTALL_FALCON_COMMAND' + env: + REGION: ${{ secrets.AWS_REGION }} + report-if-failed: name: Report if workflow failed runs-on: ubuntu-20.04 From 62320c52a6d11dd557d5e2e835a2e8e2801c1475 Mon Sep 17 00:00:00 2001 From: Alex Pickering Date: Fri, 1 Nov 2024 11:43:40 -0700 Subject: [PATCH 02/26] trigger diff in rds cf Signed-off-by: Alex Pickering --- cf/rds.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/cf/rds.yaml b/cf/rds.yaml index be9c7ffe4a..d806ce0321 100644 --- a/cf/rds.yaml +++ b/cf/rds.yaml @@ -158,3 +158,4 @@ Resources: AutoMinorVersionUpgrade: true PubliclyAccessible: false + From bb80a16958b04086de4f1abff60bb1dbdc3290c3 Mon Sep 17 00:00:00 2001 From: Alex Pickering Date: Fri, 1 Nov 2024 12:04:02 -0700 Subject: [PATCH 03/26] close quote Signed-off-by: Alex Pickering --- .github/workflows/deploy-changed-cf.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy-changed-cf.yaml b/.github/workflows/deploy-changed-cf.yaml index 9c74b03e10..7c1d34082c 100644 --- a/.github/workflows/deploy-changed-cf.yaml +++ b/.github/workflows/deploy-changed-cf.yaml @@ -461,7 +461,7 @@ jobs: aws ssm send-command --instance-ids "$INSTANCE_ID" \ --document-name AWS-RunShellScript \ - --parameters "commands='$INSTALL_FALCON_COMMAND' + --parameters "commands='$INSTALL_FALCON_COMMAND'" env: REGION: ${{ secrets.AWS_REGION }} From 8cc5485a1563dab9232d5193d1672c5c1f2c9c31 Mon Sep 17 00:00:00 2001 From: Alex Pickering Date: Fri, 1 Nov 2024 12:11:52 -0700 Subject: [PATCH 04/26] fix end of file Signed-off-by: Alex Pickering --- .github/workflows/deploy-changed-cf.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/deploy-changed-cf.yaml b/.github/workflows/deploy-changed-cf.yaml index 7c1d34082c..3ef64eda6a 100644 --- a/.github/workflows/deploy-changed-cf.yaml +++ b/.github/workflows/deploy-changed-cf.yaml @@ -462,6 +462,11 @@ jobs: aws ssm send-command --instance-ids "$INSTANCE_ID" \ --document-name AWS-RunShellScript \ --parameters "commands='$INSTALL_FALCON_COMMAND'" + + else + echo "CrowdStrike CID missing, skipping falcon sensor setup" + fi + env: REGION: ${{ secrets.AWS_REGION }} From 882a1d3b558e258a73e317fc34d8d6513f1ffce9 Mon Sep 17 00:00:00 2001 From: Alex Pickering Date: Wed, 27 Nov 2024 14:24:53 -0800 Subject: [PATCH 05/26] for staging Signed-off-by: Alex Pickering --- cf/ses.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cf/ses.yaml b/cf/ses.yaml index 6ef63a1fe1..544cf72c8a 100644 --- a/cf/ses.yaml +++ b/cf/ses.yaml @@ -48,4 +48,5 @@ Resources: Type: CNAME TTL: 1800 ResourceRecords: - - !GetAtt SESVerifiedIdentity.DkimDNSTokenValue3 \ No newline at end of file + - !GetAtt SESVerifiedIdentity.DkimDNSTokenValue3 + \ No newline at end of file From 4fb12dedbfd781a7a1f85caa87b7e1e068bb4105 Mon Sep 17 00:00:00 2001 From: Alex Pickering Date: Wed, 27 Nov 2024 14:49:25 -0800 Subject: [PATCH 06/26] try get sub Signed-off-by: Alex Pickering --- cf/sns.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cf/sns.yaml b/cf/sns.yaml index 1093df4ec9..02e996188b 100644 --- a/cf/sns.yaml +++ b/cf/sns.yaml @@ -226,3 +226,5 @@ Resources: FunctionName: !Ref SNSEndpointLambda Principal: "sns.amazonaws.com" SourceArn: !Ref SNSTopicV2 + + From 95e98351d8c318b7d7aacf0314abe5346082af05 Mon Sep 17 00:00:00 2001 From: Alex Pickering Date: Wed, 27 Nov 2024 15:03:00 -0800 Subject: [PATCH 07/26] try trigger listener rules Signed-off-by: Alex Pickering --- infra/cf-loadbalancer.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/infra/cf-loadbalancer.yaml b/infra/cf-loadbalancer.yaml index da8943a91f..850ea72325 100644 --- a/infra/cf-loadbalancer.yaml +++ b/infra/cf-loadbalancer.yaml @@ -124,3 +124,4 @@ Outputs: Value: !GetAtt LoadBalancer.LoadBalancerFullName Export: Name: !Sub 'k8s-${Environment}::LoadBalancerFullName' + From a9a0f58b55cddf50e75ab65aec92dc6ffd3bdaa5 Mon Sep 17 00:00:00 2001 From: Alex Pickering Date: Wed, 27 Nov 2024 15:03:17 -0800 Subject: [PATCH 08/26] revert sns Signed-off-by: Alex Pickering --- cf/sns.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/cf/sns.yaml b/cf/sns.yaml index 02e996188b..1093df4ec9 100644 --- a/cf/sns.yaml +++ b/cf/sns.yaml @@ -226,5 +226,3 @@ Resources: FunctionName: !Ref SNSEndpointLambda Principal: "sns.amazonaws.com" SourceArn: !Ref SNSTopicV2 - - From dff54f211de9af3bc9b084295cb5b65d711ab0ac Mon Sep 17 00:00:00 2001 From: Alex Pickering Date: Wed, 27 Nov 2024 15:05:18 -0800 Subject: [PATCH 09/26] try again Signed-off-by: Alex Pickering --- infra/cf-loadbalancer.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/infra/cf-loadbalancer.yaml b/infra/cf-loadbalancer.yaml index 850ea72325..1748a5d17a 100644 --- a/infra/cf-loadbalancer.yaml +++ b/infra/cf-loadbalancer.yaml @@ -108,6 +108,7 @@ Resources: AWS: !Join [ "", ["arn:aws:iam::", !FindInMap [ALBServiceMap, !Ref "AWS::Region", AccountId], ":root" ]] Resource: !Join [ "", [!GetAtt ALBLogsBucket.Arn, !Sub "/k8s-${Environment}/*"]] + Outputs: CanonicalHostedZoneID: Description: The ID of the Amazon Route 53 hosted zone associated with the load balancer. From b8230931f83fbbfbe4ce4cf751ea89a90e9fd713 Mon Sep 17 00:00:00 2001 From: Alex Pickering Date: Wed, 27 Nov 2024 15:25:21 -0800 Subject: [PATCH 10/26] change email Signed-off-by: Alex Pickering --- deployment-config.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment-config.json b/deployment-config.json index bd1d91a476..4fdeba04a1 100644 --- a/deployment-config.json +++ b/deployment-config.json @@ -42,7 +42,7 @@ "id": 160782110667, "region": "us-east-1" }, - "support_email": "ccbhelp@hms.harvard.edu", + "support_email": "alex_pickering@hms.harvard.edu", "reply_email": "alex_pickering@hms.harvard.edu", "deployment_environments": { "HMS": [ From d9f0fa0558094c20fb5adf7a2bc02f9f7f57702b Mon Sep 17 00:00:00 2001 From: Alex Pickering Date: Wed, 27 Nov 2024 15:45:21 -0800 Subject: [PATCH 11/26] update eksctl install Signed-off-by: Alex Pickering --- .github/workflows/deploy-infra.yaml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/deploy-infra.yaml b/.github/workflows/deploy-infra.yaml index 9c39d8f779..f1d40e10c6 100644 --- a/.github/workflows/deploy-infra.yaml +++ b/.github/workflows/deploy-infra.yaml @@ -256,7 +256,12 @@ jobs: - id: install-eksctl name: Install eksctl run: |- - curl --silent --location "https://github.com/weaveworks/eksctl/releases/latest/download/eksctl_$(uname -s)_amd64.tar.gz" | tar xz -C /tmp + ARCH=amd64 + PLATFORM=$(uname -s)_$ARCH + + curl -sLO "https://github.com/eksctl-io/eksctl/releases/latest/download/eksctl_$PLATFORM.tar.gz" + tar -xzf eksctl_$PLATFORM.tar.gz -C /tmp && rm eksctl_$PLATFORM.tar.gz + sudo mv /tmp/eksctl /usr/local/bin - id: deploy-load-balancer-role From b3898b12c2f6c6abfa127fcf3303ceadc8c86d50 Mon Sep 17 00:00:00 2001 From: Alex Pickering Date: Wed, 27 Nov 2024 15:50:00 -0800 Subject: [PATCH 12/26] update balancer role Signed-off-by: Alex Pickering --- .github/workflows/deploy-infra.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy-infra.yaml b/.github/workflows/deploy-infra.yaml index f1d40e10c6..c47218caab 100644 --- a/.github/workflows/deploy-infra.yaml +++ b/.github/workflows/deploy-infra.yaml @@ -267,7 +267,7 @@ jobs: - id: deploy-load-balancer-role name: Deploy permissions for AWS load balancer controller run: |- - curl -o iam-policy.json https://raw.githubusercontent.com/kubernetes-sigs/aws-load-balancer-controller/v2.7.2/docs/install/iam_policy.json + curl -o iam-policy.json https://raw.githubusercontent.com/kubernetes-sigs/aws-load-balancer-controller/v2.10.1/docs/install/iam_policy.json aws iam create-policy \ --policy-name AWSLoadBalancerControllerIAMPolicy-$CLUSTER_ENV \ --policy-document file://iam-policy.json || true From ca92efeede141f1bb759321eb83f70394468eece Mon Sep 17 00:00:00 2001 From: Alex Pickering Date: Wed, 27 Nov 2024 16:49:02 -0800 Subject: [PATCH 13/26] trigger rds Signed-off-by: Alex Pickering --- cf/rds.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/cf/rds.yaml b/cf/rds.yaml index be9c7ffe4a..d806ce0321 100644 --- a/cf/rds.yaml +++ b/cf/rds.yaml @@ -158,3 +158,4 @@ Resources: AutoMinorVersionUpgrade: true PubliclyAccessible: false + From 4f9469ed6f5a359cdabcb33984eb5353f4f0163f Mon Sep 17 00:00:00 2001 From: Alex Pickering Date: Wed, 27 Nov 2024 17:11:03 -0800 Subject: [PATCH 14/26] move where need Signed-off-by: Alex Pickering --- {migrations/config => config}/bucketNames.js | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename {migrations/config => config}/bucketNames.js (100%) diff --git a/migrations/config/bucketNames.js b/config/bucketNames.js similarity index 100% rename from migrations/config/bucketNames.js rename to config/bucketNames.js From e0da96a8eafdcf149b651ff3321317da97c7acfb Mon Sep 17 00:00:00 2001 From: Alex Pickering Date: Wed, 27 Nov 2024 17:38:19 -0800 Subject: [PATCH 15/26] restore orig Signed-off-by: Alex Pickering --- {config => migrations/config}/bucketNames.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename {config => migrations/config}/bucketNames.js (88%) diff --git a/config/bucketNames.js b/migrations/config/bucketNames.js similarity index 88% rename from config/bucketNames.js rename to migrations/config/bucketNames.js index b93bf6210c..16aada3da5 100644 --- a/config/bucketNames.js +++ b/migrations/config/bucketNames.js @@ -9,7 +9,7 @@ const bucketNames = { FILTERED_CELLS: `biomage-filtered-cells-${clusterEnv}-${awsAccountId}`, WORKER_RESULTS: `worker-results-${clusterEnv}-${awsAccountId}`, PLOTS: `plots-tables-${clusterEnv}-${awsAccountId}`, - CELL_METADATA: `cellenics-cell-level-metadata-${clusterEnv}-${awsAccountId}` + CELL_LEVEL_META: `cellenics-cell-level-metadata-${clusterEnv}-${awsAccountId}` }; module.exports = bucketNames; From 4dcfb2f054d98712f8ca12d38d2b11bddaf0cbe6 Mon Sep 17 00:00:00 2001 From: Alex Pickering Date: Wed, 27 Nov 2024 18:23:45 -0800 Subject: [PATCH 16/26] batch job def Signed-off-by: Alex Pickering --- cf/batch-job-definition.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cf/batch-job-definition.yaml b/cf/batch-job-definition.yaml index 71013998c0..05ad986028 100644 --- a/cf/batch-job-definition.yaml +++ b/cf/batch-job-definition.yaml @@ -45,4 +45,7 @@ Resources: Value: 2 - Type: MEMORY Value: 8192 # MiB - Type: Container \ No newline at end of file + Type: Container + + + \ No newline at end of file From 45b76a3f3cfdbb905c474c8d690bf87a85817b9e Mon Sep 17 00:00:00 2001 From: Alex Pickering Date: Wed, 27 Nov 2024 18:28:59 -0800 Subject: [PATCH 17/26] again Signed-off-by: Alex Pickering --- cf/batch-job-definition.yaml | 2 +- cf/rds.yaml | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/cf/batch-job-definition.yaml b/cf/batch-job-definition.yaml index 05ad986028..b7cad63da3 100644 --- a/cf/batch-job-definition.yaml +++ b/cf/batch-job-definition.yaml @@ -27,6 +27,7 @@ Parameters: Conditions: isPipelinePRStaged: !And [ !Equals [!Ref Environment, "staging"], !Not [ !Equals [!Ref PipelinePRNum, "" ]]] + Resources: BatchPipelineJob: Type: AWS::Batch::JobDefinition @@ -48,4 +49,3 @@ Resources: Type: Container - \ No newline at end of file diff --git a/cf/rds.yaml b/cf/rds.yaml index d806ce0321..99d8940c1f 100644 --- a/cf/rds.yaml +++ b/cf/rds.yaml @@ -156,6 +156,4 @@ Resources: DBClusterIdentifier: !Ref DBCluster Engine: aurora-postgresql AutoMinorVersionUpgrade: true - PubliclyAccessible: false - - + PubliclyAccessible: false \ No newline at end of file From f68b081653c7b10c97f3b706c7e227701a77095b Mon Sep 17 00:00:00 2001 From: Alex Pickering Date: Thu, 28 Nov 2024 09:22:34 -0800 Subject: [PATCH 18/26] revert changes --- cf/rds.yaml | 3 ++- cf/ses.yaml | 3 +-- deployment-config.json | 2 +- infra/cf-loadbalancer.yaml | 2 -- 4 files changed, 4 insertions(+), 6 deletions(-) diff --git a/cf/rds.yaml b/cf/rds.yaml index 99d8940c1f..0c690723d4 100644 --- a/cf/rds.yaml +++ b/cf/rds.yaml @@ -156,4 +156,5 @@ Resources: DBClusterIdentifier: !Ref DBCluster Engine: aurora-postgresql AutoMinorVersionUpgrade: true - PubliclyAccessible: false \ No newline at end of file + PubliclyAccessible: false + \ No newline at end of file diff --git a/cf/ses.yaml b/cf/ses.yaml index 544cf72c8a..6ef63a1fe1 100644 --- a/cf/ses.yaml +++ b/cf/ses.yaml @@ -48,5 +48,4 @@ Resources: Type: CNAME TTL: 1800 ResourceRecords: - - !GetAtt SESVerifiedIdentity.DkimDNSTokenValue3 - \ No newline at end of file + - !GetAtt SESVerifiedIdentity.DkimDNSTokenValue3 \ No newline at end of file diff --git a/deployment-config.json b/deployment-config.json index 4fdeba04a1..bd1d91a476 100644 --- a/deployment-config.json +++ b/deployment-config.json @@ -42,7 +42,7 @@ "id": 160782110667, "region": "us-east-1" }, - "support_email": "alex_pickering@hms.harvard.edu", + "support_email": "ccbhelp@hms.harvard.edu", "reply_email": "alex_pickering@hms.harvard.edu", "deployment_environments": { "HMS": [ diff --git a/infra/cf-loadbalancer.yaml b/infra/cf-loadbalancer.yaml index 1748a5d17a..da8943a91f 100644 --- a/infra/cf-loadbalancer.yaml +++ b/infra/cf-loadbalancer.yaml @@ -108,7 +108,6 @@ Resources: AWS: !Join [ "", ["arn:aws:iam::", !FindInMap [ALBServiceMap, !Ref "AWS::Region", AccountId], ":root" ]] Resource: !Join [ "", [!GetAtt ALBLogsBucket.Arn, !Sub "/k8s-${Environment}/*"]] - Outputs: CanonicalHostedZoneID: Description: The ID of the Amazon Route 53 hosted zone associated with the load balancer. @@ -125,4 +124,3 @@ Outputs: Value: !GetAtt LoadBalancer.LoadBalancerFullName Export: Name: !Sub 'k8s-${Environment}::LoadBalancerFullName' - From bea6edba45fc0c3b0b420603be3154a321d3c23f Mon Sep 17 00:00:00 2001 From: Alex Pickering Date: Thu, 28 Nov 2024 09:38:01 -0800 Subject: [PATCH 19/26] revert more --- cf/rds.yaml | 3 +-- migrations/config/bucketNames.js | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/cf/rds.yaml b/cf/rds.yaml index 0c690723d4..99d8940c1f 100644 --- a/cf/rds.yaml +++ b/cf/rds.yaml @@ -156,5 +156,4 @@ Resources: DBClusterIdentifier: !Ref DBCluster Engine: aurora-postgresql AutoMinorVersionUpgrade: true - PubliclyAccessible: false - \ No newline at end of file + PubliclyAccessible: false \ No newline at end of file diff --git a/migrations/config/bucketNames.js b/migrations/config/bucketNames.js index 16aada3da5..b93bf6210c 100644 --- a/migrations/config/bucketNames.js +++ b/migrations/config/bucketNames.js @@ -9,7 +9,7 @@ const bucketNames = { FILTERED_CELLS: `biomage-filtered-cells-${clusterEnv}-${awsAccountId}`, WORKER_RESULTS: `worker-results-${clusterEnv}-${awsAccountId}`, PLOTS: `plots-tables-${clusterEnv}-${awsAccountId}`, - CELL_LEVEL_META: `cellenics-cell-level-metadata-${clusterEnv}-${awsAccountId}` + CELL_METADATA: `cellenics-cell-level-metadata-${clusterEnv}-${awsAccountId}` }; module.exports = bucketNames; From 3213f65ae2b5e69deefc490a31772ec37c7cdec5 Mon Sep 17 00:00:00 2001 From: Alex Pickering Date: Thu, 28 Nov 2024 09:39:19 -0800 Subject: [PATCH 20/26] fix revert --- cf/rds.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cf/rds.yaml b/cf/rds.yaml index 99d8940c1f..a0f56a149c 100644 --- a/cf/rds.yaml +++ b/cf/rds.yaml @@ -156,4 +156,4 @@ Resources: DBClusterIdentifier: !Ref DBCluster Engine: aurora-postgresql AutoMinorVersionUpgrade: true - PubliclyAccessible: false \ No newline at end of file + PubliclyAccessible: false From cb7e32588f9c3c6a8c68ab984accde1cd09c322d Mon Sep 17 00:00:00 2001 From: Alex Pickering Date: Thu, 28 Nov 2024 09:40:23 -0800 Subject: [PATCH 21/26] try again --- cf/rds.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/cf/rds.yaml b/cf/rds.yaml index a0f56a149c..be9c7ffe4a 100644 --- a/cf/rds.yaml +++ b/cf/rds.yaml @@ -157,3 +157,4 @@ Resources: Engine: aurora-postgresql AutoMinorVersionUpgrade: true PubliclyAccessible: false + From 3050ea78e7387e53d475d2e3d7f0b00f94dbed58 Mon Sep 17 00:00:00 2001 From: Alex Pickering Date: Thu, 28 Nov 2024 09:56:44 -0800 Subject: [PATCH 22/26] update msg --- .github/workflows/deploy-changed-cf.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy-changed-cf.yaml b/.github/workflows/deploy-changed-cf.yaml index 236c7e48e0..b864d74e63 100644 --- a/.github/workflows/deploy-changed-cf.yaml +++ b/.github/workflows/deploy-changed-cf.yaml @@ -239,7 +239,7 @@ jobs: capabilities: "CAPABILITY_IAM,CAPABILITY_NAMED_IAM,CAPABILITY_AUTO_EXPAND" - id: deploy-batch-job-definition-template - name: Deploy CloudFormation PostRegisterLambda template + name: Deploy CloudFormation batch-job-definition template if: ${{ matrix.template == 'cf/batch-job-definition.yaml' }} uses: aws-actions/aws-cloudformation-github-deploy@v1 with: From e89ba298210e965ad3dd6debbd36168b1623802c Mon Sep 17 00:00:00 2001 From: Alex Pickering Date: Thu, 5 Dec 2024 17:06:43 -0800 Subject: [PATCH 23/26] revert changes Signed-off-by: Alex Pickering --- cf/batch-job-definition.yaml | 5 +---- cf/rds.yaml | 1 + 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/cf/batch-job-definition.yaml b/cf/batch-job-definition.yaml index b7cad63da3..71013998c0 100644 --- a/cf/batch-job-definition.yaml +++ b/cf/batch-job-definition.yaml @@ -27,7 +27,6 @@ Parameters: Conditions: isPipelinePRStaged: !And [ !Equals [!Ref Environment, "staging"], !Not [ !Equals [!Ref PipelinePRNum, "" ]]] - Resources: BatchPipelineJob: Type: AWS::Batch::JobDefinition @@ -46,6 +45,4 @@ Resources: Value: 2 - Type: MEMORY Value: 8192 # MiB - Type: Container - - + Type: Container \ No newline at end of file diff --git a/cf/rds.yaml b/cf/rds.yaml index be9c7ffe4a..d806ce0321 100644 --- a/cf/rds.yaml +++ b/cf/rds.yaml @@ -158,3 +158,4 @@ Resources: AutoMinorVersionUpgrade: true PubliclyAccessible: false + From 389cb67e9d0de54f8c03bfd84ebc0314d8c53491 Mon Sep 17 00:00:00 2001 From: hms-dbmi-cellenics Date: Fri, 6 Dec 2024 01:35:37 +0000 Subject: [PATCH 24/26] Update from https://github.com/hms-dbmi-cellenics/api/commit/c92d879bc54a44a4a9a4e0a03ce7e18e06b9ccad --- ...41016171424_add_seurat_spatial_technology.js | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 migrations/sql/20241016171424_add_seurat_spatial_technology.js diff --git a/migrations/sql/20241016171424_add_seurat_spatial_technology.js b/migrations/sql/20241016171424_add_seurat_spatial_technology.js new file mode 100644 index 0000000000..d4ae07c7a2 --- /dev/null +++ b/migrations/sql/20241016171424_add_seurat_spatial_technology.js @@ -0,0 +1,17 @@ +/** + * @param { import("knex").Knex } knex + * @returns { Promise } + */ +exports.up = async (knex) => { + // enums that are created when UI receives upload of seurat spatial object + await knex.raw("ALTER TYPE sample_technology ADD VALUE 'seurat_spatial_object';"); + await knex.raw("ALTER TYPE sample_file_type ADD VALUE 'seurat_spatial_object';"); + + // pipeline creates a ome zarr zip file for each sample in the seurat spatial object + await knex.raw("ALTER TYPE sample_file_type ADD VALUE 'ome_zarr_zip';"); +}; + +/** + * @returns { Promise < void> } +*/ +exports.down = async () => { }; From 4b3efbb845d93387c536761953d43d79fa68555c Mon Sep 17 00:00:00 2001 From: Alex Pickering Date: Fri, 6 Dec 2024 13:52:14 -0800 Subject: [PATCH 25/26] reduce max attempts Signed-off-by: Alex Pickering --- .github/workflows/deploy-infra.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy-infra.yaml b/.github/workflows/deploy-infra.yaml index c47218caab..dacac41460 100644 --- a/.github/workflows/deploy-infra.yaml +++ b/.github/workflows/deploy-infra.yaml @@ -288,7 +288,7 @@ jobs: uses: nick-invision/retry@v2 with: timeout_seconds: 600 - max_attempts: 20 + max_attempts: 10 retry_on: error on_retry_command: sleep $(shuf -i 5-15 -n 1) command: |- From b15049bb0f6d7595ecd0a070baa9198101f40a0b Mon Sep 17 00:00:00 2001 From: Alex Pickering Date: Fri, 6 Dec 2024 13:52:32 -0800 Subject: [PATCH 26/26] reduce more Signed-off-by: Alex Pickering --- .github/workflows/deploy-infra.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy-infra.yaml b/.github/workflows/deploy-infra.yaml index dacac41460..64566c2d71 100644 --- a/.github/workflows/deploy-infra.yaml +++ b/.github/workflows/deploy-infra.yaml @@ -288,7 +288,7 @@ jobs: uses: nick-invision/retry@v2 with: timeout_seconds: 600 - max_attempts: 10 + max_attempts: 5 retry_on: error on_retry_command: sleep $(shuf -i 5-15 -n 1) command: |-