From e7a0e420953b4e62c5acaa0757bc3fef92c80168 Mon Sep 17 00:00:00 2001 From: Josh Date: Tue, 8 Oct 2024 00:35:49 -0700 Subject: [PATCH 01/13] remove lambda suffix - 2 functions --- .github/workflows/cicd-pipeline.yml | 2 ++ ..._harmony_results_to_s3.py => copy_harmony_output_to_s3.py} | 2 +- terraform/lambda_functions.tf | 4 ++-- 3 files changed, 5 insertions(+), 3 deletions(-) rename bignbit/{copy_harmony_results_to_s3.py => copy_harmony_output_to_s3.py} (98%) diff --git a/.github/workflows/cicd-pipeline.yml b/.github/workflows/cicd-pipeline.yml index be6b71c..e384835 100644 --- a/.github/workflows/cicd-pipeline.yml +++ b/.github/workflows/cicd-pipeline.yml @@ -150,6 +150,7 @@ jobs: - name: Validate Terraform working-directory: terraform run: | + terraform --version terraform init -backend=false -upgrade terraform validate -no-color - name: SonarCloud Scan @@ -299,6 +300,7 @@ jobs: - name: Validate Terraform working-directory: examples/cumulus-tf run: | + terraform --version terraform init -backend=false -upgrade terraform validate -no-color diff --git a/bignbit/copy_harmony_results_to_s3.py b/bignbit/copy_harmony_output_to_s3.py similarity index 98% rename from bignbit/copy_harmony_results_to_s3.py rename to bignbit/copy_harmony_output_to_s3.py index 1bba519..9c6d3ea 100644 --- a/bignbit/copy_harmony_results_to_s3.py +++ b/bignbit/copy_harmony_output_to_s3.py @@ -9,7 +9,7 @@ from bignbit import utils -CUMULUS_LOGGER = CumulusLogger('copy_harmony_output_to_s3') +CUMULUS_LOGGER = CumulusLogger('harmony_output_to_s3') class CMA(Process): diff --git a/terraform/lambda_functions.tf b/terraform/lambda_functions.tf index 6c5d947..159bc02 100644 --- a/terraform/lambda_functions.tf +++ b/terraform/lambda_functions.tf @@ -292,7 +292,7 @@ resource "aws_lambda_function" "copy_harmony_output_to_s3" { image_config { command = ["bignbit.copy_harmony_output_to_s3.lambda_handler"] } - function_name = "${local.lambda_resources_name}-copy_harmony_output_to_s3-lambda" + function_name = "${local.lambda_resources_name}-copy_harmony_output_to_s3" role = var.lambda_role.arn timeout = 30 memory_size = 256 @@ -325,7 +325,7 @@ resource "aws_lambda_function" "apply_opera_treatment" { image_config { command = ["bignbit.apply_opera_treatment.lambda_handler"] } - function_name = "${local.lambda_resources_name}-apply_opera_treatment-lambda" + function_name = "${local.lambda_resources_name}-apply_opera_treatment" role = var.lambda_role.arn timeout = 30 memory_size = 512 From cb34c3b5d68c76842ea9f5bcd9b19ce86b37784b Mon Sep 17 00:00:00 2001 From: Josh Date: Tue, 8 Oct 2024 01:24:59 -0700 Subject: [PATCH 02/13] upload_ecr_image for all lambdas 1 --- .github/workflows/cicd-pipeline.yml | 2 +- terraform/lambda_functions.tf | 15 +++++++-------- terraform/state_machine_definition.tpl | 4 ++-- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/.github/workflows/cicd-pipeline.yml b/.github/workflows/cicd-pipeline.yml index e384835..5ec9e8d 100644 --- a/.github/workflows/cicd-pipeline.yml +++ b/.github/workflows/cicd-pipeline.yml @@ -29,7 +29,7 @@ on: env: POETRY_VERSION: "1.8.2" PYTHON_VERSION: "3.10" - TERRAFORM_VERSION: "1.3.7" + TERRAFORM_VERSION: "1.5.7" REGISTRY: ghcr.io jobs: build: diff --git a/terraform/lambda_functions.tf b/terraform/lambda_functions.tf index 159bc02..de800b1 100644 --- a/terraform/lambda_functions.tf +++ b/terraform/lambda_functions.tf @@ -48,14 +48,13 @@ resource null_resource upload_ecr_image { # This doesn't work in terraform 0.13.x because it tries to resolve the image during the plan phase instead of the # apply phase. Once IA updates to a newer terraform version, this can be used instead of having every lambda # function depend on the null_resource.upload_ecr_image resource. -#data aws_ecr_image lambda_image { -# depends_on = [ -# null_resource.upload_ecr_image -# ] -# repository_name = aws_ecr_repository.lambda-image-repo.name -# image_tag = local.ecr_image_tag -# -#} +data "aws_ecr_image" "lambda_image" { + depends_on = [ + null_resource.upload_ecr_image + ] + repository_name = aws_ecr_repository.lambda-image-repo.name + image_tag = local.ecr_image_tag +} resource "aws_lambda_function" "get_dataset_configuration" { depends_on = [ diff --git a/terraform/state_machine_definition.tpl b/terraform/state_machine_definition.tpl index e12438e..5b707f4 100644 --- a/terraform/state_machine_definition.tpl +++ b/terraform/state_machine_definition.tpl @@ -320,7 +320,7 @@ { "Variable":"$.payload.harmony_job_status", "StringMatches":"successful", - "Next":"Copy Harmony Results to S3", + "Next":"Copy Harmony Output to S3", "Comment":"Job successful" }, { @@ -344,7 +344,7 @@ ], "Default":"Wait 20 Seconds" }, - "Copy Harmony Results to S3":{ + "Copy Harmony Output to S3":{ "Type":"Task", "Resource":"${CopyHarmonyOutputToS3Lambda}", "Parameters":{ From 4a405744aa285f771b21e7bb6c1ec491628c998d Mon Sep 17 00:00:00 2001 From: Josh Date: Tue, 8 Oct 2024 12:41:54 -0700 Subject: [PATCH 03/13] terraform version 1.5.3 --- .github/workflows/cicd-pipeline.yml | 2 +- bignbit/copy_harmony_output_to_s3.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cicd-pipeline.yml b/.github/workflows/cicd-pipeline.yml index 5ec9e8d..01f4627 100644 --- a/.github/workflows/cicd-pipeline.yml +++ b/.github/workflows/cicd-pipeline.yml @@ -29,7 +29,7 @@ on: env: POETRY_VERSION: "1.8.2" PYTHON_VERSION: "3.10" - TERRAFORM_VERSION: "1.5.7" + TERRAFORM_VERSION: "1.5.3" REGISTRY: ghcr.io jobs: build: diff --git a/bignbit/copy_harmony_output_to_s3.py b/bignbit/copy_harmony_output_to_s3.py index 9c6d3ea..1bba519 100644 --- a/bignbit/copy_harmony_output_to_s3.py +++ b/bignbit/copy_harmony_output_to_s3.py @@ -9,7 +9,7 @@ from bignbit import utils -CUMULUS_LOGGER = CumulusLogger('harmony_output_to_s3') +CUMULUS_LOGGER = CumulusLogger('copy_harmony_output_to_s3') class CMA(Process): From 0cb0f17103a4f8b996129a1985cda3526c0fb642 Mon Sep 17 00:00:00 2001 From: Josh Date: Tue, 8 Oct 2024 15:48:19 -0700 Subject: [PATCH 04/13] /version 0.1.1rc5 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 94458af..bb002ad 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "bignbit" -version = "0.1.1rc4" +version = "0.1.1rc5" description = "Browse image generation and transfer" authors = ["PO.DAAC "] license = "Apache 2.0" From da15af84d87989005a25783b5c043b5d6deeed3a Mon Sep 17 00:00:00 2001 From: Josh Date: Wed, 9 Oct 2024 13:18:24 -0700 Subject: [PATCH 05/13] trim lambda function names --- .github/workflows/cicd-pipeline.yml | 4 +-- pyproject.toml | 2 +- terraform/lambda_functions.tf | 41 +++++++++++++++-------------- 3 files changed, 23 insertions(+), 24 deletions(-) diff --git a/.github/workflows/cicd-pipeline.yml b/.github/workflows/cicd-pipeline.yml index 01f4627..be6b71c 100644 --- a/.github/workflows/cicd-pipeline.yml +++ b/.github/workflows/cicd-pipeline.yml @@ -29,7 +29,7 @@ on: env: POETRY_VERSION: "1.8.2" PYTHON_VERSION: "3.10" - TERRAFORM_VERSION: "1.5.3" + TERRAFORM_VERSION: "1.3.7" REGISTRY: ghcr.io jobs: build: @@ -150,7 +150,6 @@ jobs: - name: Validate Terraform working-directory: terraform run: | - terraform --version terraform init -backend=false -upgrade terraform validate -no-color - name: SonarCloud Scan @@ -300,7 +299,6 @@ jobs: - name: Validate Terraform working-directory: examples/cumulus-tf run: | - terraform --version terraform init -backend=false -upgrade terraform validate -no-color diff --git a/pyproject.toml b/pyproject.toml index bb002ad..94458af 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "bignbit" -version = "0.1.1rc5" +version = "0.1.1rc4" description = "Browse image generation and transfer" authors = ["PO.DAAC "] license = "Apache 2.0" diff --git a/terraform/lambda_functions.tf b/terraform/lambda_functions.tf index de800b1..119bc5b 100644 --- a/terraform/lambda_functions.tf +++ b/terraform/lambda_functions.tf @@ -5,9 +5,9 @@ locals { ecr_image_name_and_tag = split(":", element(local.lambda_container_image_uri_split, length(local.lambda_container_image_uri_split) - 1)) ecr_image_name = "${local.environment}-${element(local.ecr_image_name_and_tag, 0)}" ecr_image_tag = element(local.ecr_image_name_and_tag, 1) - build_image_sets_function_name = "${local.lambda_resources_name}-build_image_sets-lambda" - send_to_gitc_function_name = "${local.lambda_resources_name}-send_to_gitc-lambda" - handle_gitc_response_function_name = "${local.lambda_resources_name}-handle_gitc_response-lambda" + build_image_sets_function_name = "${local.lambda_resources_name}-build_image_sets" + send_to_gitc_function_name = "${local.lambda_resources_name}-send_to_gitc" + handle_gitc_response_function_name = "${local.lambda_resources_name}-handle_gitc_response" } resource aws_ecr_repository "lambda-image-repo" { @@ -48,13 +48,14 @@ resource null_resource upload_ecr_image { # This doesn't work in terraform 0.13.x because it tries to resolve the image during the plan phase instead of the # apply phase. Once IA updates to a newer terraform version, this can be used instead of having every lambda # function depend on the null_resource.upload_ecr_image resource. -data "aws_ecr_image" "lambda_image" { - depends_on = [ - null_resource.upload_ecr_image - ] - repository_name = aws_ecr_repository.lambda-image-repo.name - image_tag = local.ecr_image_tag -} +#data aws_ecr_image lambda_image { +# depends_on = [ +# null_resource.upload_ecr_image +# ] +# repository_name = aws_ecr_repository.lambda-image-repo.name +# image_tag = local.ecr_image_tag +# +#} resource "aws_lambda_function" "get_dataset_configuration" { depends_on = [ @@ -66,7 +67,7 @@ resource "aws_lambda_function" "get_dataset_configuration" { image_config { command = ["bignbit.get_dataset_configuration.lambda_handler"] } - function_name = "${local.lambda_resources_name}-get_dataset_configuration-lambda" + function_name = "${local.lambda_resources_name}-get_dataset_configuration" role = var.lambda_role.arn timeout = 30 memory_size = 256 @@ -97,7 +98,7 @@ resource "aws_lambda_function" "get_granule_umm_json" { image_config { command = ["bignbit.get_granule_umm_json.lambda_handler"] } - function_name = "${local.lambda_resources_name}-get_granule_umm_json-lambda" + function_name = "${local.lambda_resources_name}-get_granule_umm_json" role = var.lambda_role.arn timeout = 30 memory_size = 256 @@ -130,7 +131,7 @@ resource "aws_lambda_function" "get_collection_concept_id" { image_config { command = ["bignbit.get_collection_concept_id.lambda_handler"] } - function_name = "${local.lambda_resources_name}-get_collection_concept_id-lambda" + function_name = "${local.lambda_resources_name}-get_collection_concept_id" role = var.lambda_role.arn timeout = 30 memory_size = 256 @@ -163,7 +164,7 @@ resource "aws_lambda_function" "identify_image_file" { image_config { command = ["bignbit.identify_image_file.lambda_handler"] } - function_name = "${local.lambda_resources_name}-identify_image_file-lambda" + function_name = "${local.lambda_resources_name}-identify_image_file" role = var.lambda_role.arn timeout = 30 memory_size = 256 @@ -194,7 +195,7 @@ resource "aws_lambda_function" "submit_harmony_job" { image_config { command = ["bignbit.submit_harmony_job.lambda_handler"] } - function_name = "${local.lambda_resources_name}-submit_harmony_job-lambda" + function_name = "${local.lambda_resources_name}-submit_harmony_job" role = var.lambda_role.arn timeout = 30 memory_size = 256 @@ -227,7 +228,7 @@ resource "aws_lambda_function" "generate_image_metadata" { image_config { command = ["bignbit.generate_image_metadata.lambda_handler"] } - function_name = "${local.lambda_resources_name}-generate_image_metadata-lambda" + function_name = "${local.lambda_resources_name}-generate_image_metadata" role = var.lambda_role.arn timeout = 30 memory_size = 256 @@ -258,7 +259,7 @@ resource "aws_lambda_function" "get_harmony_job_status" { image_config { command = ["bignbit.get_harmony_job_status.lambda_handler"] } - function_name = "${local.lambda_resources_name}-get_harmony_job_status-lambda" + function_name = "${local.lambda_resources_name}-get_harmony_job_status" role = var.lambda_role.arn timeout = 30 memory_size = 256 @@ -393,7 +394,7 @@ resource "aws_lambda_function" "send_to_gitc" { image_config { command = ["bignbit.send_to_gitc.lambda_handler"] } - function_name = "${local.lambda_resources_name}-send_to_gitc-lambda" + function_name = "${local.lambda_resources_name}-send_to_gitc" role = var.lambda_role.arn timeout = 15 memory_size = 128 @@ -427,7 +428,7 @@ resource "aws_lambda_function" "handle_gitc_response" { image_config { command = ["bignbit.handle_gitc_response.handler"] } - function_name = "${local.lambda_resources_name}-handle_gitc_response-lambda" + function_name = "${local.lambda_resources_name}-handle_gitc_response" role = var.lambda_role.arn timeout = 5 memory_size = 128 @@ -458,7 +459,7 @@ resource "aws_lambda_function" "save_cma_message" { image_config { command = ["bignbit.save_cma_message.lambda_handler"] } - function_name = "${local.lambda_resources_name}-save_cma_message-lambda" + function_name = "${local.lambda_resources_name}-save_cma_message" role = var.lambda_role.arn timeout = 15 memory_size = 128 From 43b7ab6eb982a37f634c2028bf9aac461fbe85a0 Mon Sep 17 00:00:00 2001 From: Josh Date: Thu, 10 Oct 2024 15:27:50 -0700 Subject: [PATCH 06/13] SWOT lambdas fix --- CHANGELOG.md | 9 +++++++++ examples/README.md | 4 ++-- pyproject.toml | 2 +- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6e9e6e0..7c7ed47 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed ### Security +## [0.1.2] +### Added +### Changed +- BIG terraform failing in SWOT venues due to long function(lambda) names +### Deprecated +### Removed +### Fixed +### Security + ## [0.1.1] ### Added diff --git a/examples/README.md b/examples/README.md index aca7798..2b79491 100644 --- a/examples/README.md +++ b/examples/README.md @@ -1,8 +1,8 @@ # BIGnBIT Module In Cumulus Deployment (cumulus-tf) - BIGnBIT Module used in `cumulus-deploy-tf/cumulus-tf` - Module Name: `big_and_bit_module` - - Source ZIP: `https://github.com/podaac/bignbit/releases/download/0.1.1/bignbit-0.1.1-cumulus-tf.zip` - - lambda_container_image_uri: `ghcr.io/podaac/bignbit/bignbit:0.1.1` + - Source ZIP: `https://github.com/podaac/bignbit/releases/download/0.1.2/bignbit-0.1.2-cumulus-tf.zip` + - lambda_container_image_uri: `ghcr.io/podaac/bignbit/bignbit:0.1.2` ## Terraform ZIP diff --git a/pyproject.toml b/pyproject.toml index 94458af..211bd18 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "bignbit" -version = "0.1.1rc4" +version = "0.1.2rc1" description = "Browse image generation and transfer" authors = ["PO.DAAC "] license = "Apache 2.0" From 09f6d01d5915ecc3e885457b472843fb8968f665 Mon Sep 17 00:00:00 2001 From: voxparcxls Date: Thu, 10 Oct 2024 22:31:21 +0000 Subject: [PATCH 07/13] /version 0.1.2rc2 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 211bd18..8846443 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "bignbit" -version = "0.1.2rc1" +version = "0.1.2rc2" description = "Browse image generation and transfer" authors = ["PO.DAAC "] license = "Apache 2.0" From ffcecd4f019cdf2ccdf6f119654d962191b2fd09 Mon Sep 17 00:00:00 2001 From: Josh Haile <35245966+voxparcxls@users.noreply.github.com> Date: Thu, 10 Oct 2024 16:05:39 -0700 Subject: [PATCH 08/13] update release (#44) --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 8846443..211bd18 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "bignbit" -version = "0.1.2rc2" +version = "0.1.2rc1" description = "Browse image generation and transfer" authors = ["PO.DAAC "] license = "Apache 2.0" From 99ca5525df795cf7749b50b4dc485e18ea943905 Mon Sep 17 00:00:00 2001 From: James Wood Date: Mon, 25 Nov 2024 15:16:09 -0800 Subject: [PATCH 09/13] update version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 211bd18..67b5fe2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "bignbit" -version = "0.1.2rc1" +version = "0.1.2rc3" description = "Browse image generation and transfer" authors = ["PO.DAAC "] license = "Apache 2.0" From e9ae6c5896b8dfcb4453a7f306e81b3371d0b546 Mon Sep 17 00:00:00 2001 From: James Wood Date: Mon, 25 Nov 2024 15:17:20 -0800 Subject: [PATCH 10/13] deploy From ad96c109896ed32fcc9f2dbfa19ac0cfe0f6dbcc Mon Sep 17 00:00:00 2001 From: James Wood Date: Mon, 25 Nov 2024 15:18:58 -0800 Subject: [PATCH 11/13] update changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7c7ed47..ea728bd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,7 +15,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [0.1.2] ### Added ### Changed -- BIG terraform failing in SWOT venues due to long function(lambda) names +- BIG terraform failing in SWOT venues due to long function(lambda) names ### Deprecated ### Removed ### Fixed From bfe8198f613a2de435efeeb36976d0ec72436d59 Mon Sep 17 00:00:00 2001 From: jamesfwood Date: Mon, 25 Nov 2024 23:21:46 +0000 Subject: [PATCH 12/13] /version 0.1.2rc4 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 67b5fe2..ff890fc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "bignbit" -version = "0.1.2rc3" +version = "0.1.2rc4" description = "Browse image generation and transfer" authors = ["PO.DAAC "] license = "Apache 2.0" From 42234ecc7530c2775f63fedca8e5edfd0400c467 Mon Sep 17 00:00:00 2001 From: jamesfwood Date: Mon, 25 Nov 2024 23:37:46 +0000 Subject: [PATCH 13/13] /version 0.1.2rc5 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index ff890fc..cc25421 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "bignbit" -version = "0.1.2rc4" +version = "0.1.2rc5" description = "Browse image generation and transfer" authors = ["PO.DAAC "] license = "Apache 2.0"