Merge pull request #810 from GoogleCloudPlatform/jccb/recursive-tests

Fully recursive e2e test runner for examples

blueprints/data-solutions/data-platform-foundations/README.md

@@ -222,7 +222,7 @@ module "data-platform" {
   prefix              = "myprefix"
 }
 
-# tftest modules=1 resources=1
+# tftest modules=42 resources=314
 ```
 
 ## Customizations

blueprints/gke/multitenant-fleet/README.md

@@ -95,7 +95,7 @@ module "gke" {
     }
   }
 }
-# tftest modules=1 resources=0
+# tftest modules=5 resources=26
 ```
 
 ## Creating Multiple Clusters
@@ -185,7 +185,7 @@ module "gke" {
     }
   }
 }
-# tftest modules=1 resources=0
+# tftest modules=7 resources=28
 ```
 
 ## Multiple clusters with GKE Fleet
@@ -305,7 +305,7 @@ module "gke" {
   }
 }
 
-# tftest modules=1 resources=0
+# tftest modules=8 resources=39
 ```
 
 <!-- TFDOC OPTS files:1 show_extra:1 -->

modules/compute-vm/README.md

@@ -247,7 +247,7 @@ module "vm-with-gvnic" {
   }
   service_account_create = true
 }
-# tftest modules=1 resources=2
+# tftest modules=1 resources=3
 ```
 
 ### Instance template

modules/net-glb/README.md

@@ -154,7 +154,7 @@ resource "google_compute_region_network_endpoint_group" "serverless-neg" {
     service = "my-cloud-run-service"
   }
 }
-# tftest modules=1 resources=4
+# tftest modules=1 resources=5
 ```
 
 ### Mixing Backends
@@ -478,7 +478,7 @@ resource "tls_self_signed_cert" "self_signed_cert" {
     organization = "My Test Org"
   }
 }
-# tftest modules=1 resources=6
+# tftest modules=1 resources=8
 ```
 
 ## Regional Load Balancing

modules/net-ilb-l7/README.md

@@ -158,7 +158,7 @@ resource "google_compute_network_endpoint_group" "my-neg" {
   default_port = "90"
   zone         = "europe-west1-b"
 }
-# tftest modules=1 resources=5
+# tftest modules=1 resources=6
 ```
 -->
 
@@ -367,7 +367,7 @@ resource "tls_self_signed_cert" "self_signed_cert" {
     organization = "My Test Org"
   }
 }
-# tftest modules=1 resources=6
+# tftest modules=1 resources=8
 ```
 
 ## Components And Files Mapping

tests/conftest.py

@@ -88,18 +88,28 @@ def run_plan(fixture_path=None, targets=None, refresh=True,
 
 
 @pytest.fixture(scope='session')
-def doc_example_plan_runner(_plan_runner):
-  "Returns a function to run Terraform plan on documentation examples."
-
-  def run_plan(fixture_path=None):
-    "Runs Terraform plan and returns count of modules and resources."
-    tf = tftest.TerraformTest(fixture_path, BASEDIR,
-                              os.environ.get('TERRAFORM', 'terraform'))
-    tf.setup(upgrade=True)
-    plan = tf.plan(output=True, refresh=True)
-    # the fixture is the example we are testing
-    modules = plan.modules or {}
-    return (len(modules), sum(len(m.resources) for m in modules.values()))
+def recursive_e2e_plan_runner(_plan_runner):
+  """Plan runner for end-to-end root module, returns total number of
+  (nested) modules and resources"""
+
+  def walk_plan(node, modules, resources):
+    # TODO(jccb): this would be better with node.get() but
+    # TerraformPlanOutput objects don't have it
+    new_modules = node['child_modules'] if 'child_modules' in node else []
+    resources += node['resources'] if 'resources' in node else []
+    modules += new_modules
+    for module in new_modules:
+      walk_plan(module, modules, resources)
+
+  def run_plan(fixture_path=None, targets=None, refresh=True,
+               include_bare_resources=False, compute_sums=True, **tf_vars):
+    "Runs Terraform plan on a root module using defaults, returns data."
+    plan = _plan_runner(fixture_path, targets=targets, refresh=refresh,
+                        **tf_vars)
+    modules = []
+    resources = []
+    walk_plan(plan.root_module, modules, resources)
+    return len(modules), len(resources)
 
   return run_plan
 

tests/examples/test_plan.py

@@ -19,7 +19,7 @@
 EXPECTED_RESOURCES_RE = re.compile(r'# tftest modules=(\d+) resources=(\d+)')
 
 
-def test_example(doc_example_plan_runner, tmp_path, example):
+def test_example(recursive_e2e_plan_runner, tmp_path, example):
   (tmp_path / 'fabric').symlink_to(Path(BASE_PATH, '../../').resolve())
   (tmp_path / 'variables.tf').symlink_to(
       Path(BASE_PATH, 'variables.tf').resolve())
@@ -29,6 +29,6 @@ def test_example(doc_example_plan_runner, tmp_path, example):
   expected_modules = int(match.group(1)) if match is not None else 1
   expected_resources = int(match.group(2)) if match is not None else 1
 
-  num_modules, num_resources = doc_example_plan_runner(str(tmp_path))
+  num_modules, num_resources = recursive_e2e_plan_runner(str(tmp_path))
   assert expected_modules == num_modules
   assert expected_resources == num_resources

tests/fast/stages/s00_bootstrap/test_plan.py

@@ -26,8 +26,8 @@
 # }
 
 
-def test_counts(fast_e2e_plan_runner):
+def test_counts(recursive_e2e_plan_runner):
   "Test stage."
   # TODO: to re-enable per-module resource count check print _, then test
-  num_modules, num_resources, _ = fast_e2e_plan_runner()
+  num_modules, num_resources = recursive_e2e_plan_runner()
   assert num_modules > 0 and num_resources > 0

tests/fast/stages/s01_resman/test_plan.py

@@ -13,8 +13,8 @@
 # limitations under the License.
 
 
-def test_counts(fast_e2e_plan_runner):
+def test_counts(recursive_e2e_plan_runner):
   "Test stage."
-  num_modules, num_resources, _ = fast_e2e_plan_runner()
+  num_modules, num_resources = recursive_e2e_plan_runner()
   # TODO: to re-enable per-module resource count check print _, then test
   assert num_modules > 0 and num_resources > 0

tests/fast/stages/s02_networking_nva/test_plan.py

@@ -13,8 +13,8 @@
 # limitations under the License.
 
 
-def test_counts(fast_e2e_plan_runner):
+def test_counts(recursive_e2e_plan_runner):
   "Test stage."
-  num_modules, num_resources, _ = fast_e2e_plan_runner()
+  num_modules, num_resources = recursive_e2e_plan_runner()
   # TODO: to re-enable per-module resource count check print _, then test
   assert num_modules > 0 and num_resources > 0

tests/fast/stages/s02_networking_peering/test_plan.py

@@ -27,9 +27,9 @@
 STAGE_VPN = STAGES / '02-networking-vpn'
 
 
-def test_counts(fast_e2e_plan_runner):
+def test_counts(recursive_e2e_plan_runner):
   'Test stage.'
-  num_modules, num_resources, _ = fast_e2e_plan_runner()
+  num_modules, num_resources = recursive_e2e_plan_runner()
   # TODO: to re-enable per-module resource count check print _, then test
   assert num_modules > 0 and num_resources > 0
 

tests/fast/stages/s02_networking_vpn/test_plan.py

@@ -13,8 +13,8 @@
 # limitations under the License.
 
 
-def test_counts(fast_e2e_plan_runner):
+def test_counts(recursive_e2e_plan_runner):
   "Test stage."
-  num_modules, num_resources, _ = fast_e2e_plan_runner()
+  num_modules, num_resources = recursive_e2e_plan_runner()
   # TODO: to re-enable per-module resource count check print _, then test
   assert num_modules > 0 and num_resources > 0

tests/fast/stages/s02_security/test_plan.py

@@ -13,8 +13,8 @@
 # limitations under the License.
 
 
-def test_counts(fast_e2e_plan_runner):
+def test_counts(recursive_e2e_plan_runner):
   "Test stage."
-  num_modules, num_resources, _ = fast_e2e_plan_runner()
+  num_modules, num_resources = recursive_e2e_plan_runner()
   # TODO: to re-enable per-module resource count check print _, then test
   assert num_modules > 0 and num_resources > 0

tests/fast/stages/s03_data_platform/test_plan.py

@@ -13,8 +13,8 @@
 # limitations under the License.
 
 
-def test_counts(fast_e2e_plan_runner):
+def test_counts(recursive_e2e_plan_runner):
   "Test stage."
-  num_modules, num_resources, _ = fast_e2e_plan_runner()
+  num_modules, num_resources = recursive_e2e_plan_runner()
   # TODO: to re-enable per-module resource count check print _, then test
   assert num_modules > 0 and num_resources > 0

tests/fast/stages/s03_gke_multitenant/test_plan.py

@@ -13,8 +13,8 @@
 # limitations under the License.
 
 
-def test_counts(fast_e2e_plan_runner):
+def test_counts(recursive_e2e_plan_runner):
   "Test stage."
-  num_modules, num_resources, _ = fast_e2e_plan_runner()
+  num_modules, num_resources = recursive_e2e_plan_runner()
   # TODO: to re-enable per-module resource count check print _, then test
   assert num_modules > 0 and num_resources > 0

tests/fast/stages/s03_project_factory/test_plan.py

@@ -13,8 +13,8 @@
 # limitations under the License.
 
 
-def test_counts(fast_e2e_plan_runner):
+def test_counts(recursive_e2e_plan_runner):
   "Test stage."
-  num_modules, num_resources, _ = fast_e2e_plan_runner()
+  num_modules, num_resources = recursive_e2e_plan_runner()
   # TODO: to re-enable per-module resource count check print _, then test
   assert num_modules > 0 and num_resources > 0