test: Add new test skeleton based on Terratest and add/extend tests for multiple modules

.gitignore

@@ -32,7 +32,7 @@
 .terragrunt-cache
 .vscode
 .idea
-
+**/test_report.html
 # Palo auth codes
 authcodes
 # Crash log files

go.mod

@@ -1,11 +1,58 @@
 module github.com/PaloAltoNetworks/terraform-aws-vmseries-modules
 
-go 1.14
+go 1.18
 
 require (
-	github.com/gruntwork-io/terratest v0.35.7
-	github.com/hashicorp/go-version v1.6.0 // indirect
+	github.com/gruntwork-io/terratest v0.41.7
 	github.com/hashicorp/terraform-json v0.14.0
-	github.com/stretchr/testify v1.4.0
+	github.com/stretchr/testify v1.8.1
+	golang.org/x/exp v0.0.0-20230124195608-d38c7dcee874
+)
+
+require (
+	cloud.google.com/go v0.83.0 // indirect
+	cloud.google.com/go/storage v1.10.0 // indirect
+	github.com/agext/levenshtein v1.2.3 // indirect
+	github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect
+	github.com/aws/aws-sdk-go v1.40.56 // indirect
+	github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/golang/snappy v0.0.3 // indirect
+	github.com/googleapis/gax-go/v2 v2.0.5 // indirect
+	github.com/hashicorp/errwrap v1.0.0 // indirect
+	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
+	github.com/hashicorp/go-getter v1.6.1 // indirect
+	github.com/hashicorp/go-multierror v1.1.0 // indirect
+	github.com/hashicorp/go-safetemp v1.0.0 // indirect
+	github.com/hashicorp/go-version v1.6.0 // indirect
+	github.com/hashicorp/hcl/v2 v2.9.1 // indirect
+	github.com/jinzhu/copier v0.0.0-20190924061706-b57f9002281a // indirect
+	github.com/jmespath/go-jmespath v0.4.0 // indirect
+	github.com/jstemmer/go-junit-report v0.9.1 // indirect
+	github.com/klauspost/compress v1.13.0 // indirect
+	github.com/mattn/go-zglob v0.0.2-0.20190814121620-e3c945676326 // indirect
+	github.com/mitchellh/go-homedir v1.1.0 // indirect
+	github.com/mitchellh/go-testing-interface v1.0.0 // indirect
+	github.com/mitchellh/go-wordwrap v1.0.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/tmccombs/hcl2json v0.3.3 // indirect
+	github.com/ulikunitz/xz v0.5.8 // indirect
 	github.com/zclconf/go-cty v1.11.0 // indirect
+	go.opencensus.io v0.23.0 // indirect
+	golang.org/x/crypto v0.1.0 // indirect
+	golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 // indirect
+	golang.org/x/mod v0.6.0 // indirect
+	golang.org/x/net v0.1.0 // indirect
+	golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c // indirect
+	golang.org/x/sys v0.1.0 // indirect
+	golang.org/x/text v0.4.0 // indirect
+	golang.org/x/tools v0.2.0 // indirect
+	google.golang.org/api v0.47.0 // indirect
+	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c // indirect
+	google.golang.org/grpc v1.38.0 // indirect
+	google.golang.org/protobuf v1.26.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

modules/vmseries/main.tf

@@ -89,7 +89,6 @@ resource "aws_instance" "this" {
     delete_on_termination = true
     encrypted             = var.ebs_encrypted
     kms_key_id            = var.ebs_encrypted == false ? null : data.aws_kms_alias.current_arn[0].target_key_arn
-    tags                  = merge(var.tags, { Name = var.name })
   }
 
   # Attach primary interface to the instance
@@ -104,6 +103,11 @@ resource "aws_instance" "this" {
 
   tags = merge(var.tags, { Name = var.name })
 
+  # If volume_tags are not defined, then module is NOT idempotent. If after deployment terraform plan is executed,
+  # then update in-place is planned for resource "aws_instance" "this" with below change:
+  # + volume_tags = {}
+  volume_tags = merge(var.tags, { Name = var.name })
+
   lifecycle {
     ignore_changes = [
       user_data,

scripts/install.sh

@@ -1,4 +1,4 @@
-#!/usr/bin/bash
+#!/bin/bash
 
 # install.sh - prepare the dependencies for the run.sh
 # 

scripts/run.sh

@@ -1,4 +1,4 @@
-#!/usr/bin/bash
+#!/bin/bash
 
 # run.sh - Run the usual pre-commit checks.
 

tests/README.md

@@ -0,0 +1,69 @@
+# Quick start
+
+## How to execute tests
+
+Testing Terraform modules:
+1. Install required binaries:
+    * Terraform at the specific version that you'd like to test: https://developer.hashicorp.com/terraform
+    * Go at the latest 1.* version: https://golang.org/
+2. Configuration authentication settings e.g. use https://github.com/Nike-Inc/gimme-aws-creds or set ``AWS_REGION`` environment variable and also ``AWS_ACCESS_KEY_ID``, ``AWS_SECRET_ACCESS_KEY``, or similar.
+3. Get ``terratest`` package by running command:
+```bash
+go get -u github.com/gruntwork-io/terratest
+```
+4. Execute test for module using commands e.g for ``bootstrap`` module:
+```bash
+cd tests/bootstrap
+go test -v -timeout 30m -count=1
+```
+
+Run all test:
+
+```bash
+go test -timeout 130m ./... -json | go-test-report
+```
+Comments:
+* Do not however run `go test -v .` or similar. Specifying a package (that extra dot) enables caching, which is incompatible with Terraform.
+* We use go-test-report to create html reports for tests, check https://github.com/vakenbolt/go-test-report for more information
+* Cloud resources are destroyed automatically after the test, no cleanup is normally required.
+* VScode users should keep `Go: Test On Save` at the default false value, and not set to true. This option is spelled `go.testOnSave` in settings.json.
+
+## Test skeleton overview
+
+```mermaid
+graph TB
+    terraform_options(Init Terraform with provided options)
+    terraform_apply(Deploy infrastructure)
+    do_terraform_plan_after_deploy{Execute Terraform Plan?}
+    terraform_plan_after_deploy(Verify if no changes are planned after deployment)
+    do_modify_infrastructure{Modify infrastructure?}
+    modify_infrastructure(Plan infrastructure with changed resources)
+    verify_changes(Verify planned changes)
+    terraform_apply_changes(Deploy infrastructure with changed resources)
+    verify_assert_expression(Verify assert expressions)
+    terraform_destroy(Destroy infrastructure)
+    test_fail((Tests failed))
+    test_pass((Tests passed))
+
+    terraform_options --> terraform_apply -- infrastructure is deployed --> verify_assert_expression
+    do_terraform_plan_after_deploy -- yes --> terraform_plan_after_deploy
+    terraform_plan_after_deploy -- code is idempotent --> do_modify_infrastructure
+    verify_assert_expression -- all asserts passed --> do_terraform_plan_after_deploy
+    do_modify_infrastructure -- yes --> modify_infrastructure --> verify_changes
+    verify_changes -- only expected changes --> terraform_apply_changes
+    terraform_apply -. error in deployment .-> test_fail
+    terraform_plan_after_deploy -. code is not idempotent .-> test_fail
+    verify_assert_expression -. one of the asserts failed .-> test_fail
+    verify_changes -. unexpected changes .-> test_fail
+    terraform_apply_changes -. error in deployment .-> test_fail
+    do_terraform_plan_after_deploy -- no --> test_pass
+    do_modify_infrastructure -- no --> test_pass
+    terraform_apply_changes -- infrastructure is deployed --> test_pass
+    test_fail -..-> terraform_destroy
+    test_pass ----> terraform_destroy
+
+    classDef green fill:#33aa33,stroke:#333,stroke-width:2px;
+    classDef red fill:#aa3333,stroke:#333,stroke-width:2px;
+    class test_pass green
+    class test_fail red
+```

tests/alb/main.tf

@@ -0,0 +1,102 @@
+## VPC
+
+module "security_vpc" {
+  source = "../../modules/vpc"
+
+  name                    = "${var.name_prefix}-vpc"
+  cidr_block              = var.security_vpc_cidr
+  security_groups         = var.security_vpc_security_groups
+  create_internet_gateway = true
+  enable_dns_hostnames    = true
+  enable_dns_support      = true
+  instance_tenancy        = "default"
+}
+
+module "security_subnet_sets" {
+  source = "../../modules/subnet_set"
+
+  for_each = toset(distinct([for _, v in var.security_vpc_subnets : v.set]))
+
+  name                = each.key
+  vpc_id              = module.security_vpc.id
+  has_secondary_cidrs = module.security_vpc.has_secondary_cidrs
+  cidrs               = { for k, v in var.security_vpc_subnets : k => v if v.set == each.key }
+}
+
+locals {
+  security_vpc_routes = concat(
+    [for cidr in ["app_vm", "app_lb"] :
+      {
+        subnet_key   = cidr
+        next_hop_set = module.security_vpc.igw_as_next_hop_set
+        to_cidr      = "0.0.0.0/0"
+      }
+    ]
+  )
+}
+module "security_vpc_routes" {
+  for_each = { for route in local.security_vpc_routes : "${route.subnet_key}_${route.to_cidr}" => route }
+  source   = "../../modules/vpc_route"
+
+  route_table_ids = module.security_subnet_sets[each.value.subnet_key].unique_route_table_ids
+  to_cidr         = each.value.to_cidr
+  next_hop_set    = each.value.next_hop_set
+}
+
+## ALB
+
+module "public_alb" {
+  source = "../../modules/alb"
+
+  lb_name         = replace("${var.name_prefix}${var.application_lb_name}", "_", "-")
+  subnets         = { for k, v in module.security_subnet_sets["app_vm"].subnets : k => { id = v.id } }
+  vpc_id          = module.security_vpc.id
+  security_groups = [module.security_vpc.security_group_ids["app_vm"]]
+  rules           = var.application_lb_rules
+  targets         = { for k, v in var.app_vms : k => aws_instance.app_vm[k].private_ip }
+
+  tags = var.global_tags
+}
+
+
+### app EC2 instance ###
+
+data "aws_ami" "this" {
+  most_recent = true # newest by time, not by version number
+
+  filter {
+    name   = "name"
+    values = ["bitnami-nginx-1.21*-linux-debian-10-x86_64-hvm-ebs-nami"]
+    # The wildcard '*' causes re-creation of the whole EC2 instance when a new image appears.
+  }
+
+  owners = ["979382823631"] # bitnami = 979382823631
+}
+
+resource "tls_private_key" "random_ssh_key" {
+  algorithm = "RSA"
+  rsa_bits  = 4096
+}
+
+resource "aws_key_pair" "random_ssh_key_pair" {
+  key_name   = var.key_pair_name
+  public_key = tls_private_key.random_ssh_key.public_key_openssh
+}
+
+resource "aws_instance" "app_vm" {
+  for_each = var.app_vms
+
+  ami                         = data.aws_ami.this.id
+  instance_type               = var.app_vm_type
+  key_name                    = aws_key_pair.random_ssh_key_pair.key_name
+  subnet_id                   = module.security_subnet_sets["app_vm"].subnets[each.value.az].id
+  vpc_security_group_ids      = [module.security_vpc.security_group_ids["app_vm"]]
+  tags                        = merge({ Name = "${var.name_prefix}${each.key}" }, var.global_tags)
+  associate_public_ip_address = true
+
+}
+
+data "aws_network_interface" "bar" {
+  for_each = var.app_vms
+  id       = aws_instance.app_vm[each.key].primary_network_interface_id
+}

tests/alb/main_test.go

@@ -0,0 +1,93 @@
+package main
+
+import (
+	"log"
+	"testing"
+
+	"github.com/PaloAltoNetworks/terraform-aws-vmseries-modules/tests/internal/helpers"
+	"github.com/PaloAltoNetworks/terraform-aws-vmseries-modules/tests/internal/testskeleton"
+	"github.com/gruntwork-io/terratest/modules/logger"
+	"github.com/gruntwork-io/terratest/modules/terraform"
+)
+
+func TestALBOutputAndConectivitiyWithFullTFVars(t *testing.T) {
+
+	// define variables for Terraform
+	namePrefix := "terratest-alb-"
+
+	// define options for Terraform
+	terraformOptions := terraform.WithDefaultRetryableErrors(t, &terraform.Options{
+		TerraformDir: ".",
+		VarFiles:     []string{"terraform_full.tfvars"},
+		Vars: map[string]interface{}{
+			"name_prefix": namePrefix,
+		},
+		Logger:               logger.Default,
+		Lock:                 true,
+		Upgrade:              true,
+		SetVarsAfterVarFiles: true,
+	})
+
+	destroyFunc := func() {
+		terraform.Destroy(t, terraformOptions)
+	}
+	defer destroyFunc()
+	terraformOptions = testskeleton.DeployInfraNoCheckOutputsNoDestroy(t, terraformOptions)
+
+	albName := terraform.Output(t, terraformOptions, "alb_name")
+	log.Printf("Alb_name = %s", albName)
+
+	assertList := []testskeleton.AssertExpression{
+		// check if the ALB is created with correct FQDN
+		{
+			OutputName: "alb_name",
+			Operation:  testskeleton.NotEmpty,
+		},
+		// check if the ALB is created with correct FQDN
+		{
+			OutputName:    "alb_name",
+			Operation:     testskeleton.StartsWith,
+			ExpectedValue: namePrefix,
+		},
+		// check communication with app
+		{
+			Operation:   testskeleton.CheckFunctionWithValue,
+			Check:       helpers.CheckHttpGetWebApp,
+			TestedValue: "http://" + albName + "/",
+		},
+	}
+	testskeleton.AssertOutputs(t, terraformOptions, assertList)
+
+}
+
+func TestALBOutputWithMinimumTFVars(t *testing.T) {
+
+	// define variables for Terraform
+	namePrefix := "terratest-alb-"
+	// define options for Terraform
+	terraformOptions := terraform.WithDefaultRetryableErrors(t, &terraform.Options{
+		TerraformDir: ".",
+		VarFiles:     []string{"terraform_minimum.tfvars"},
+		Vars: map[string]interface{}{
+			"name_prefix": namePrefix,
+		},
+		Logger:               logger.Default,
+		Lock:                 true,
+		Upgrade:              true,
+		SetVarsAfterVarFiles: true,
+	})
+	assertList := []testskeleton.AssertExpression{
+		// check if the ALB is created with correct FQDN
+		{
+			OutputName: "alb_name",
+			Operation:  testskeleton.NotEmpty,
+		},
+		// check if the ALB is created with correct FQDN
+		{
+			OutputName:    "alb_name",
+			Operation:     testskeleton.StartsWith,
+			ExpectedValue: namePrefix,
+		},
+	}
+	testskeleton.DeployInfraCheckOutputs(t, terraformOptions, assertList)
+}

tests/alb/outputs.tf

@@ -0,0 +1,7 @@
+output "alb_name" {
+  value = module.public_alb.lb_fqdn
+}
+
+output "vms_public_ips" {
+  value = [for k, v in var.app_vms : aws_instance.app_vm[k].public_ip]
+}