From 6dc0321580876afb3705beba1f152e8b70fd1389 Mon Sep 17 00:00:00 2001 From: Sunny Date: Mon, 8 Jan 2024 15:25:21 +0000 Subject: [PATCH] tests/int: Add separate resource cleanup step Introduce a destroy-only mode in the test runner to run terraform destroy for the respective cloud provider configurations. This can be used to destroy cloud resources without going through the whole provision-test process. Add a new step in github actions workflow to run the test binary in destoy-only mode at the very end irrespective of the result of the previous steps. This ensures that the infrastructure is always destroyed, even if the CI job is cancelled. Signed-off-by: Sunny --- .github/workflows/integration-azure.yaml | 9 ++++++ .github/workflows/integration-gcp.yaml | 7 +++++ tests/integration/Makefile | 12 ++++++++ tests/integration/README.md | 18 ++++++++++-- tests/integration/go.mod | 4 +-- tests/integration/go.sum | 4 +-- tests/integration/suite_test.go | 37 ++++++++++++++++++------ 7 files changed, 75 insertions(+), 16 deletions(-) diff --git a/.github/workflows/integration-azure.yaml b/.github/workflows/integration-azure.yaml index 6db3bb13..db6fe790 100644 --- a/.github/workflows/integration-azure.yaml +++ b/.github/workflows/integration-azure.yaml @@ -70,3 +70,12 @@ jobs: ARM_SUBSCRIPTION_ID: ${{ secrets.IRC_E2E_AZ_ARM_SUBSCRIPTION_ID }} ARM_TENANT_ID: ${{ secrets.IRC_E2E_AZ_ARM_TENANT_ID }} TF_VAR_azure_location: ${{ vars.TF_VAR_azure_location }} + - name: Ensure resource cleanup + if: ${{ always() }} + run: . .env && make destroy-azure + env: + ARM_CLIENT_ID: ${{ secrets.IRC_E2E_AZ_ARM_CLIENT_ID }} + ARM_CLIENT_SECRET: ${{ secrets.IRC_E2E_AZ_ARM_CLIENT_SECRET }} + ARM_SUBSCRIPTION_ID: ${{ secrets.IRC_E2E_AZ_ARM_SUBSCRIPTION_ID }} + ARM_TENANT_ID: ${{ secrets.IRC_E2E_AZ_ARM_TENANT_ID }} + TF_VAR_azure_location: ${{ vars.TF_VAR_azure_location }} diff --git a/.github/workflows/integration-gcp.yaml b/.github/workflows/integration-gcp.yaml index 2b21d9c5..6d9fdfb4 100644 --- a/.github/workflows/integration-gcp.yaml +++ b/.github/workflows/integration-gcp.yaml @@ -84,3 +84,10 @@ jobs: TF_VAR_gcp_project_id: ${{ vars.TF_VAR_gcp_project_id }} TF_VAR_gcp_region: ${{ vars.TF_VAR_gcp_region }} TF_VAR_gcp_zone: ${{ vars.TF_VAR_gcp_zone }} + - name: Ensure resource cleanup + if: ${{ always() }} + run: . .env && make destroy-gcp + env: + TF_VAR_gcp_project_id: ${{ vars.TF_VAR_gcp_project_id }} + TF_VAR_gcp_region: ${{ vars.TF_VAR_gcp_region }} + TF_VAR_gcp_zone: ${{ vars.TF_VAR_gcp_zone }} diff --git a/tests/integration/Makefile b/tests/integration/Makefile index 024c74fd..9a510564 100644 --- a/tests/integration/Makefile +++ b/tests/integration/Makefile @@ -32,3 +32,15 @@ test-azure: test-gcp: $(MAKE) test PROVIDER_ARG="-provider gcp" + +destroy: + go test -timeout $(TEST_TIMEOUT) -v $(GO_TEST_PATH) $(GO_TEST_ARGS) $(PROVIDER_ARG) -destroy-only + +destroy-aws: + $(MAKE) destroy PROVIDER_ARG="-provider aws" + +destroy-azure: + $(MAKE) destroy PROVIDER_ARG="-provider azure" + +destroy-gcp: + $(MAKE) destroy PROVIDER_ARG="-provider gcp" diff --git a/tests/integration/README.md b/tests/integration/README.md index f28c4c1a..ef2d6922 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -49,6 +49,10 @@ variables using use the terraform configuration below. Please make sure all the requirements of azure-gh-actions are followed before running it. +**NOTE:** When running the following for a repo under an organization, set the +environment variable `GITHUB_ORGANIZATION` if setting the `owner` in the +`github` provider doesn't work. + ```hcl provider "github" { owner = "fluxcd" @@ -149,6 +153,10 @@ variables using use the terraform configuration below. Please make sure all the requirements of gcp-gh-actions are followed before running it. +**NOTE:** When running the following for a repo under an organization, set the +environment variable `GITHUB_ORGANIZATION` if setting the `owner` in the +`github` provider doesn't work. + ```hcl provider "google" {} @@ -256,9 +264,13 @@ using the initial `build/flux.yaml` manifest. Once the environment is ready, the individual go tests are executed. After the tests end, the environment is destroyed automatically. -**IMPORTANT**: In case the terraform infrastructure results in a bad state, -maybe due to a crash during the apply, the whole infrastructure can be destroyed -by running `terraform destroy` in `terraform/` directory. +If not configured explicitly to retain the infrastructure, at the end of the +test, the test infrastructure is deleted. In case of any failure due to which +the resources don't get deleted, the `make destroy-*` commands can be run for +the respective provider. This will run terraform destroy in the respective +provider's terraform configuration directory. This can be used to quickly +destroy the infrastructure without going through the provision-test-destroy +steps. ## Debugging the tests diff --git a/tests/integration/go.mod b/tests/integration/go.mod index f650406c..314a3698 100644 --- a/tests/integration/go.mod +++ b/tests/integration/go.mod @@ -6,7 +6,8 @@ replace github.com/fluxcd/image-reflector-controller/api => ../../api require ( github.com/fluxcd/image-reflector-controller/api v0.0.0 - github.com/fluxcd/test-infra/tftestenv v0.0.0-20230530120643-bdcf7573fb2f + github.com/fluxcd/test-infra/tftestenv v0.0.0-20240108135005-b58e0c4e0cfa + github.com/hashicorp/terraform-exec v0.18.1 github.com/hashicorp/terraform-json v0.15.0 github.com/onsi/gomega v1.30.0 k8s.io/apimachinery v0.28.4 @@ -40,7 +41,6 @@ require ( github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/go-version v1.6.0 // indirect github.com/hashicorp/hc-install v0.5.0 // indirect - github.com/hashicorp/terraform-exec v0.18.1 // indirect github.com/imdario/mergo v0.3.12 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect diff --git a/tests/integration/go.sum b/tests/integration/go.sum index 6bbc0734..27f658de 100644 --- a/tests/integration/go.sum +++ b/tests/integration/go.sum @@ -185,8 +185,8 @@ github.com/fluxcd/pkg/apis/acl v0.1.0 h1:EoAl377hDQYL3WqanWCdifauXqXbMyFuK82NnX6 github.com/fluxcd/pkg/apis/acl v0.1.0/go.mod h1:zfEZzz169Oap034EsDhmCAGgnWlcWmIObZjYMusoXS8= github.com/fluxcd/pkg/apis/meta v1.2.0 h1:O766PzGAdMdQKybSflGL8oV0+GgCNIkdsxfalRyzeO8= github.com/fluxcd/pkg/apis/meta v1.2.0/go.mod h1:fU/Az9AoVyIxC0oI4ihG0NVMNnvrcCzdEym3wxjIQsc= -github.com/fluxcd/test-infra/tftestenv v0.0.0-20230530120643-bdcf7573fb2f h1:aTwZxfVUtm15HyAtgK1kfNTVvyeRoP4oUjDmEh5FZcY= -github.com/fluxcd/test-infra/tftestenv v0.0.0-20230530120643-bdcf7573fb2f/go.mod h1:liFlLEXgambGVdWSJ4JzbIHf1Vjpp1HwUyPazPIVZug= +github.com/fluxcd/test-infra/tftestenv v0.0.0-20240108135005-b58e0c4e0cfa h1:JdI+rVwGF5gBYt+UBijOVzXtq7aAU80vgksMNXSCCfU= +github.com/fluxcd/test-infra/tftestenv v0.0.0-20240108135005-b58e0c4e0cfa/go.mod h1:liFlLEXgambGVdWSJ4JzbIHf1Vjpp1HwUyPazPIVZug= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= diff --git a/tests/integration/suite_test.go b/tests/integration/suite_test.go index 403db894..0217b330 100644 --- a/tests/integration/suite_test.go +++ b/tests/integration/suite_test.go @@ -26,6 +26,7 @@ import ( "testing" "time" + "github.com/hashicorp/terraform-exec/tfexec" tfjson "github.com/hashicorp/terraform-json" "k8s.io/apimachinery/pkg/runtime" @@ -69,6 +70,9 @@ var ( // verbose flag to enable output of terraform execution. verbose = flag.Bool("verbose", false, "verbose output of the environment setup") + // destroyOnly flag to destroy any provisioned infrastructure. + destroyOnly = flag.Bool("destroy-only", false, "run in destroy-only mode and delete any existing infrastructure") + // testRepos is a map of registry common name and URL of the test // repositories. This is used as the test cases to run the tests against. // The registry common name need not be the actual registry address but an @@ -126,15 +130,6 @@ func TestMain(m *testing.M) { flag.Parse() ctx := context.TODO() - ircImg := os.Getenv("TEST_IMG") - if ircImg == "" { - log.Fatal("TEST_IMG must be set to the test image-reflector-controller image, cannot be empty") - } - - localImgs := map[string]string{ - "image-reflector-controller": ircImg, - } - // Validate the provider. if *targetProvider == "" { log.Fatalf("-provider flag must be set to one of %v", supportedProviders) @@ -154,6 +149,30 @@ func TestMain(m *testing.M) { log.Fatalf("Failed to get provider config for %q", *targetProvider) } + // Run destroy-only mode if enabled. + if *destroyOnly { + log.Println("Running in destroy-only mode...") + envOpts := []tftestenv.EnvironmentOption{ + tftestenv.WithVerbose(*verbose), + // Ignore any state lock in destroy-only mode. + tftestenv.WithTfDestroyOptions(tfexec.Lock(false)), + } + if err := tftestenv.Destroy(ctx, providerCfg.terraformPath, envOpts...); err != nil { + panic(err) + } + os.Exit(0) + } + + // Check the test app image. + ircImg := os.Getenv("TEST_IMG") + if ircImg == "" { + log.Fatal("TEST_IMG must be set to the test image-reflector-controller image, cannot be empty") + } + + localImgs := map[string]string{ + "image-reflector-controller": ircImg, + } + // Construct scheme to be added to the kubeclient. scheme := runtime.NewScheme() err := imagev1.AddToScheme(scheme)