diff --git a/blueprints/cloud-operations/iam-delegated-role-grants/README.md b/blueprints/cloud-operations/iam-delegated-role-grants/README.md index dae32ef9bf..4c4d227da6 100644 --- a/blueprints/cloud-operations/iam-delegated-role-grants/README.md +++ b/blueprints/cloud-operations/iam-delegated-role-grants/README.md @@ -27,13 +27,12 @@ By changing the `restricted_role_grant`, the blueprint can be used to grant admi You can easily configure the blueprint for this use case: -```hcl +```tfvars # terraform.tfvars delegated_role_grants = ["roles/compute.networkUser"] -direct_role_grants = [] +direct_role_grants = [] restricted_role_grant = "roles/compute.networkAdmin" -# tftest skip ``` This diagram shows the resources and expected behaviour: diff --git a/blueprints/data-solutions/data-platform-foundations/README.md b/blueprints/data-solutions/data-platform-foundations/README.md index 51545d58e5..b7fcc806a6 100644 --- a/blueprints/data-solutions/data-platform-foundations/README.md +++ b/blueprints/data-solutions/data-platform-foundations/README.md @@ -135,7 +135,6 @@ service_encryption_keys = { storage = "KEY_URL_MULTIREGIONAL" pubsub = "KEY_URL_MULTIREGIONAL" } -# tftest skip ``` This step is optional and depends on customer policies and security best practices. @@ -198,8 +197,7 @@ billing_account_id = "111111-222222-333333" older_id = "folders/123456789012" organization_domain = "domain.com" prefix = "myco" -# tftest skip` -`` +``` For more fine details check variables on [`variables.tf`](./variables.tf) and update according to the desired configuration. Remember to create team groups described [below](#groups). diff --git a/blueprints/networking/hub-and-spoke-vpn/README.md b/blueprints/networking/hub-and-spoke-vpn/README.md index 20fb2c518d..2ba3a86ad3 100644 --- a/blueprints/networking/hub-and-spoke-vpn/README.md +++ b/blueprints/networking/hub-and-spoke-vpn/README.md @@ -35,7 +35,7 @@ You can easily create such a project by commenting turning on project creation i ```hcl module "project" { - source = "../../..//modules/project" + source = "../../../modules/project" name = var.project_id # comment or remove this line to enable project creation # project_create = false diff --git a/modules/__experimental/net-dns-policy-address/README.md b/modules/__experimental/net-dns-policy-address/README.md index 3dbabdac34..7044ef2ecc 100644 --- a/modules/__experimental/net-dns-policy-address/README.md +++ b/modules/__experimental/net-dns-policy-address/README.md @@ -12,7 +12,7 @@ module "dns-policy-addresses" { project_id = "myproject" regions = ["europe-west1", "europe-west3"] } -# tftest skip +# tftest skip (uses data sources) ``` The output is a map with lists of addresses of type `DNS_RESOLVER` for each region specified in variables. diff --git a/modules/endpoints/README.md b/modules/endpoints/README.md index 0d85961dce..4e6fd2eb94 100644 --- a/modules/endpoints/README.md +++ b/modules/endpoints/README.md @@ -11,14 +11,24 @@ module "endpoint" { source = "./fabric/modules/endpoints" project_id = "my-project" service_name = "YOUR-API.endpoints.YOUR-PROJECT-ID.cloud.goog" - openapi_config = { "yaml_path" = "openapi.yaml" } + openapi_config = { "yaml_path" = "configs/endpoints/openapi.yaml" } iam = { "servicemanagement.serviceController" = [ "serviceAccount:123456890-compute@developer.gserviceaccount.com" ] } } -# tftest skip +# tftest modules=1 resources=2 files=openapi +``` + +```yaml +# tftest file openapi configs/endpoints/openapi.yaml +swagger: "2.0" +info: + description: "A simple Google Cloud Endpoints API example." + title: "Endpoints Example" + version: "1.0.0" +host: "echo-api.endpoints.YOUR-PROJECT-ID.cloud.goog" ``` [Here](https://github.com/GoogleCloudPlatform/python-docs-samples/blob/master/endpoints/getting-started/openapi.yaml) you can find an example of an openapi.yaml file. Once created the endpoint, remember to activate the service at project level. diff --git a/modules/folder/README.md b/modules/folder/README.md index 1943882d0d..5120d24673 100644 --- a/modules/folder/README.md +++ b/modules/folder/README.md @@ -89,20 +89,19 @@ module "folder" { parent = "organizations/1234567890" name = "Folder name" firewall_policy_factory = { - cidr_file = "data/cidrs.yaml" + cidr_file = "configs/firewall-policies/cidrs.yaml" policy_name = null - rules_file = "data/rules.yaml" + rules_file = "configs/firewall-policies/rules.yaml" } firewall_policy_association = { factory-policy = module.folder.firewall_policy_id["factory"] } } -# tftest skip +# tftest modules=1 resources=5 files=cidrs,rules ``` ```yaml -# cidrs.yaml - +# tftest file cidrs configs/firewall-policies/cidrs.yaml rfc1918: - 10.0.0.0/8 - 172.16.0.0/12 @@ -110,8 +109,7 @@ rfc1918: ``` ```yaml -# rules.yaml - +# tftest file rules configs/firewall-policies/rules.yaml allow-admins: description: Access from the admin subnet to all subnets direction: INGRESS diff --git a/modules/folder/organization-policies.tf b/modules/folder/organization-policies.tf index 3766005ae5..999d1c5862 100644 --- a/modules/folder/organization-policies.tf +++ b/modules/folder/organization-policies.tf @@ -17,14 +17,10 @@ # tfdoc:file:description Folder-level organization policies. locals { - _factory_data_raw = ( - var.org_policies_data_path == null - ? tomap({}) - : merge([ - for f in fileset(var.org_policies_data_path, "*.yaml") : - yamldecode(file("${var.org_policies_data_path}/${f}")) - ]...) - ) + _factory_data_raw = merge([ + for f in try(fileset(var.org_policies_data_path, "*.yaml"), []) : + yamldecode(file("${var.org_policies_data_path}/${f}")) + ]...) # simulate applying defaults to data coming from yaml files _factory_data = { diff --git a/modules/kms/README.md b/modules/kms/README.md index cfaa822a91..4fe7485788 100644 --- a/modules/kms/README.md +++ b/modules/kms/README.md @@ -23,7 +23,7 @@ module "kms" { keyring_create = false keys = { key-a = null, key-b = null, key-c = null } } -# tftest skip +# tftest skip (uses data sources) ``` ### Keyring creation and crypto key rotation and IAM roles diff --git a/modules/net-vpc-firewall/README.md b/modules/net-vpc-firewall/README.md index 6b36edd93c..d0cd32ff03 100644 --- a/modules/net-vpc-firewall/README.md +++ b/modules/net-vpc-firewall/README.md @@ -138,16 +138,16 @@ module "firewall" { project_id = "my-project" network = "my-network" factories_config = { - rules_folder = "config/firewall" - cidr_tpl_file = "config/cidr_template.yaml" + rules_folder = "configs/firewal/rules" + cidr_tpl_file = "configs/firewal/cidr_template.yaml" } } -# tftest skip +# tftest modules=1 resources=3 ``` ```yaml -# ./config/firewall/load_balancers.yaml +# tftest file configs/firewall/rules/load_balancers.yaml allow-healthchecks: description: Allow ingress from healthchecks. ranges: @@ -161,13 +161,12 @@ allow-healthchecks: ``` ```yaml -# ./config/cidr_template.yaml +# tftest file configs/firewall/cidr_template.yaml healthchecks: - 35.191.0.0/16 - 130.211.0.0/22 - 209.85.152.0/22 - 209.85.204.0/22 - ``` diff --git a/modules/net-vpc/README.md b/modules/net-vpc/README.md index 0d6a231e11..b596940623 100644 --- a/modules/net-vpc/README.md +++ b/modules/net-vpc/README.md @@ -233,11 +233,11 @@ module "vpc" { name = "my-network" data_folder = "config/subnets" } -# tftest skip +# tftest modules=1 resources=1 file=subnets ``` ```yaml -# ./config/subnets/subnet-name.yaml +# tftest file subnets ./config/subnets/subnet-name.yaml region: europe-west1 description: Sample description ip_cidr_range: 10.0.0.0/24 diff --git a/modules/organization/README.md b/modules/organization/README.md index d2e8ef319c..a716e665b7 100644 --- a/modules/organization/README.md +++ b/modules/organization/README.md @@ -137,16 +137,15 @@ module "org" { source = "./fabric/modules/organization" organization_id = var.organization_id - org_policy_custom_constraints_data_path = "/my/path" - + org_policy_custom_constraints_data_path = "configs/custom-constraints" } -# tftest skip +# tftest modules=1 resources=3 files=gke,dataproc ``` ```yaml -# /my/path/gke.yaml +# tftest file gke configs/custom-constraints/gke.yaml custom.gkeEnableLogging: - resource_types: + resource_types: - container.googleapis.com/Cluster method_types: - CREATE @@ -155,7 +154,7 @@ custom.gkeEnableLogging: action_type: DENY display_name: Do not disable Cloud Logging custom.gkeEnableAutoUpgrade: - resource_types: + resource_types: - container.googleapis.com/NodePool method_types: - CREATE @@ -166,10 +165,9 @@ custom.gkeEnableAutoUpgrade: ``` ```yaml -# /my/path/dataproc.yaml - -custom.dataprocNoMoreThan10Workers - resource_types: +# tftest file dataproc configs/custom-constraints/dataproc.yaml +custom.dataprocNoMoreThan10Workers: + resource_types: - dataproc.googleapis.com/Cluster method_types: - CREATE @@ -228,20 +226,19 @@ module "org" { source = "./fabric/modules/organization" organization_id = var.organization_id firewall_policy_factory = { - cidr_file = "data/cidrs.yaml" + cidr_file = "configs/firewall-policies/cidrs.yaml" policy_name = null - rules_file = "data/rules.yaml" + rules_file = "configs/firewall-policies/rules.yaml" } firewall_policy_association = { factory-policy = module.org.firewall_policy_id["factory"] } } -# tftest skip +# tftest modules=1 resources=4 files=cidrs,rules ``` ```yaml -# cidrs.yaml - +# tftest file cidrs configs/firewall-policies/cidrs.yaml rfc1918: - 10.0.0.0/8 - 172.16.0.0/12 @@ -249,8 +246,7 @@ rfc1918: ``` ```yaml -# rules.yaml - +# tftest file rules configs/firewall-policies/rules.yaml allow-admins: description: Access from the admin subnet to all subnets direction: INGRESS diff --git a/modules/organization/org-policy-custom-constraints.tf b/modules/organization/org-policy-custom-constraints.tf index bfc2de1bce..6a8cf5e6d4 100644 --- a/modules/organization/org-policy-custom-constraints.tf +++ b/modules/organization/org-policy-custom-constraints.tf @@ -15,14 +15,11 @@ */ locals { - _custom_constraints_factory_data_raw = ( - var.org_policy_custom_constraints_data_path == null - ? tomap({}) - : tomap(merge([ - for f in fileset(var.org_policy_custom_constraints_data_path, "*.yaml") : - yamldecode(file("${var.org_policy_custom_constraints_data_path}/${f}")) - ]...)) - ) + _custom_constraints_factory_data_raw = merge([ + for f in try(fileset(var.org_policy_custom_constraints_data_path, "*.yaml"), []) : + yamldecode(file("${var.org_policy_custom_constraints_data_path}/${f}")) + ]...) + _custom_constraints_factory_data = { for k, v in local._custom_constraints_factory_data_raw : diff --git a/modules/organization/organization-policies.tf b/modules/organization/organization-policies.tf index 425e8f520b..62d464557a 100644 --- a/modules/organization/organization-policies.tf +++ b/modules/organization/organization-policies.tf @@ -17,14 +17,10 @@ # tfdoc:file:description Organization-level organization policies. locals { - _factory_data_raw = ( - var.org_policies_data_path == null - ? tomap({}) - : merge([ - for f in fileset(var.org_policies_data_path, "*.yaml") : - yamldecode(file("${var.org_policies_data_path}/${f}")) - ]...) - ) + _factory_data_raw = merge([ + for f in try(fileset(var.org_policies_data_path, "*.yaml"), []) : + yamldecode(file("${var.org_policies_data_path}/${f}")) + ]...) # simulate applying defaults to data coming from yaml files _factory_data = { diff --git a/modules/project/README.md b/modules/project/README.md index 03ea00a19d..b6fb3881cb 100644 --- a/modules/project/README.md +++ b/modules/project/README.md @@ -224,13 +224,13 @@ module "folder" { source = "./fabric/modules/folder" parent = "organizations/1234567890" name = "Folder name" - org_policies_data_path = "/my/path" + org_policies_data_path = "configs/org-policies/" } -# tftest skip +# tftest modules=1 resources=6 files=boolean,list ``` ```yaml -# /my/path/boolean.yaml +# tftest file boolean configs/org-policies/boolean.yaml iam.disableServiceAccountKeyCreation: enforce: true @@ -246,7 +246,7 @@ iam.disableServiceAccountKeyUpload: ``` ```yaml -# /my/path/list.yaml +# tftest file list configs/org-policies/list.yaml compute.vmExternalIpAccess: deny: all: true diff --git a/modules/project/organization-policies.tf b/modules/project/organization-policies.tf index d64dd95608..7763aff402 100644 --- a/modules/project/organization-policies.tf +++ b/modules/project/organization-policies.tf @@ -17,14 +17,10 @@ # tfdoc:file:description Project-level organization policies. locals { - _factory_data_raw = ( - var.org_policies_data_path == null - ? tomap({}) - : merge([ - for f in fileset(var.org_policies_data_path, "*.yaml") : - yamldecode(file("${var.org_policies_data_path}/${f}")) - ]...) - ) + _factory_data_raw = merge([ + for f in try(fileset(var.org_policies_data_path, "*.yaml"), []) : + yamldecode(file("${var.org_policies_data_path}/${f}")) + ]...) # simulate applying defaults to data coming from yaml files _factory_data = { diff --git a/modules/projects-data-source/README.md b/modules/projects-data-source/README.md index 35c5df5610..617a01dbd1 100644 --- a/modules/projects-data-source/README.md +++ b/modules/projects-data-source/README.md @@ -22,7 +22,7 @@ output "folders" { value = module.my-org.folders } -# tftest skip +# tftest skip (uses data sources) ``` ### My dev projects based on parent and label @@ -42,7 +42,7 @@ output "dev-folders" { value = module.my-dev.folders } -# tftest skip +# tftest skip (uses data sources) ``` diff --git a/tests/examples/conftest.py b/tests/examples/conftest.py index 2cea9e368c..b2b915ea1d 100644 --- a/tests/examples/conftest.py +++ b/tests/examples/conftest.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +import collections +import re from pathlib import Path import marko @@ -21,6 +23,11 @@ MODULES_PATH = FABRIC_ROOT / 'modules/' BLUEPRINTS_PATH = FABRIC_ROOT / 'blueprints/' +FILE_TEST_RE = re.compile(r'# tftest file (\w+) ([\S]+)') + +Example = collections.namedtuple('Example', 'code files') +File = collections.namedtuple('File', 'path content') + def pytest_generate_tests(metafunc): if 'example' in metafunc.fixturenames: @@ -37,19 +44,30 @@ def pytest_generate_tests(metafunc): doc = marko.parse(readme.read_text()) index = 0 last_header = None - mark = pytest.mark.xdist_group(name=module.name) + files = {} + + #first pass: collect all tftest tagged files + for child in doc.children: + if isinstance(child, marko.block.FencedCode): + code = child.children[0].children + match = FILE_TEST_RE.search(code) + if match: + name, path = match.groups() + files[name] = File(path, code) + for child in doc.children: - if isinstance(child, marko.block.FencedCode) and child.lang == 'hcl': + if isinstance(child, marko.block.FencedCode): index += 1 code = child.children[0].children if 'tftest skip' in code: continue - examples.append(pytest.param(code, marks=mark)) - path = module.relative_to(FABRIC_ROOT) - name = f'{path}:{last_header}' - if index > 1: - name += f' {index}' - ids.append(name) + if child.lang == 'hcl': + examples.append(Example(code, files)) + path = module.relative_to(FABRIC_ROOT) + name = f'{path}:{last_header}' + if index > 1: + name += f' {index}' + ids.append(name) elif isinstance(child, marko.block.Heading): last_header = child.children[0].children index = 0 diff --git a/tests/examples/test_plan.py b/tests/examples/test_plan.py index be0686424d..2bb45af12d 100644 --- a/tests/examples/test_plan.py +++ b/tests/examples/test_plan.py @@ -16,20 +16,30 @@ from pathlib import Path BASE_PATH = Path(__file__).parent -EXPECTED_RESOURCES_RE = re.compile(r'# tftest modules=(\d+) resources=(\d+)') +COUNT_TEST_RE = re.compile( + r'# tftest modules=(\d+) resources=(\d+)(?: files=([\w,]+))?') def test_example(recursive_e2e_plan_runner, tmp_path, example): - (tmp_path / 'fabric').symlink_to(Path(BASE_PATH, '../../').resolve()) - (tmp_path / 'variables.tf').symlink_to( - Path(BASE_PATH, 'variables.tf').resolve()) - (tmp_path / 'main.tf').write_text(example) - - match = EXPECTED_RESOURCES_RE.search(example) - expected_modules = int(match.group(1)) if match is not None else 1 - expected_resources = int(match.group(2)) if match is not None else 1 - - num_modules, num_resources = recursive_e2e_plan_runner( - str(tmp_path), tmpdir=False) - assert expected_modules == num_modules - assert expected_resources == num_resources + if match := COUNT_TEST_RE.search(example.code): + (tmp_path / 'fabric').symlink_to(Path(BASE_PATH, '../../')) + (tmp_path / 'variables.tf').symlink_to(Path(BASE_PATH, 'variables.tf')) + (tmp_path / 'main.tf').write_text(example.code) + + if match.group(3) is not None: + requested_files = match.group(3).split(',') + for f in requested_files: + destination = tmp_path / example.files[f].path + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text(example.files[f].content) + + expected_modules = int(match.group(1)) if match is not None else 1 + expected_resources = int(match.group(2)) if match is not None else 1 + + num_modules, num_resources = recursive_e2e_plan_runner( + str(tmp_path), tmpdir=False) + assert expected_modules == num_modules, 'wrong number of modules' + assert expected_resources == num_resources, 'wrong number of resources' + + else: + assert False, "can't find tftest directive" diff --git a/tests/modules/organization/test_plan_org_policies_modules.py b/tests/modules/organization/test_plan_org_policies_modules.py index 8bc3a390ba..1d19ee1eb6 100644 --- a/tests/modules/organization/test_plan_org_policies_modules.py +++ b/tests/modules/organization/test_plan_org_policies_modules.py @@ -38,8 +38,8 @@ def test_policy_implementation(): '+# tfdoc:file:description Folder-level organization policies.\n', ' \n', ' locals {\n', - ' _factory_data_raw = (\n', - '@@ -69,8 +69,8 @@\n', + ' _factory_data_raw = merge([\n', + '@@ -65,8 +65,8 @@\n', ' org_policies = {\n', ' for k, v in local._org_policies :\n', ' k => merge(v, {\n', @@ -64,8 +64,8 @@ def test_policy_implementation(): '+# tfdoc:file:description Organization-level organization policies.\n', ' \n', ' locals {\n', - ' _factory_data_raw = (\n', - '@@ -69,8 +69,8 @@\n', + ' _factory_data_raw = merge([\n', + '@@ -65,8 +65,8 @@\n', ' org_policies = {\n', ' for k, v in local._org_policies :\n', ' k => merge(v, {\n', @@ -76,7 +76,7 @@ def test_policy_implementation(): ' \n', ' is_boolean_policy = v.allow == null && v.deny == null\n', ' has_values = (\n', - '@@ -143,4 +143,13 @@\n', + '@@ -139,4 +139,13 @@\n', ' }\n', ' }\n', ' }\n',