diff --git a/blueprints/data-solutions/composer-2/README.md b/blueprints/data-solutions/composer-2/README.md index 4ee62126c9..08a8643d68 100644 --- a/blueprints/data-solutions/composer-2/README.md +++ b/blueprints/data-solutions/composer-2/README.md @@ -96,20 +96,20 @@ service_encryption_keys = { | name | description | type | required | default | |---|---|:---:|:---:|:---:| -| [prefix](variables.tf#L81) | Unique prefix used for resource names. Not used for project if 'project_create' is null. | string | ✓ | | -| [project_id](variables.tf#L95) | Project id, references existing project if `project_create` is null. | string | ✓ | | -| [composer_config](variables.tf#L17) | Composer environemnt configuration. See [attribute reference](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/composer_environment#argument-reference---cloud-composer-2) for details on settings variables. | object({…}) | | {…} | -| [iam_groups_map](variables.tf#L61) | Map of Role => groups to be added on the project. Example: { \"roles/composer.admin\" = [\"group:gcp-data-engineers@example.com\"]}. | map(list(string)) | | null | -| [network_config](variables.tf#L67) | Shared VPC network configurations to use. If null networks will be created in projects with preconfigured values. | object({…}) | | null | -| [project_create](variables.tf#L86) | Provide values if project creation is needed, uses existing project if null. Parent is in 'folders/nnn' or 'organizations/nnn' format. | object({…}) | | null | -| [region](variables.tf#L100) | Region where instances will be deployed. | string | | "europe-west1" | -| [service_encryption_keys](variables.tf#L106) | Cloud KMS keys to use to encrypt resources. Provide a key for each reagion in use. | map(string) | | null | +| [prefix](variables.tf#L78) | Unique prefix used for resource names. Not used for project if 'project_create' is null. | string | ✓ | | +| [project_id](variables.tf#L92) | Project id, references existing project if `project_create` is null. | string | ✓ | | +| [composer_config](variables.tf#L17) | Composer environment configuration. It accepts only following attributes: `environment_size`, `software_config` and `workloads_config`. See [attribute reference](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/composer_environment#argument-reference---cloud-composer-2) for details on settings variables. | object({…}) | | {…} | +| [iam_groups_map](variables.tf#L58) | Map of Role => groups to be added on the project. Example: { \"roles/composer.admin\" = [\"group:gcp-data-engineers@example.com\"]}. | map(list(string)) | | null | +| [network_config](variables.tf#L64) | Shared VPC network configurations to use. If null networks will be created in projects with preconfigured values. | object({…}) | | null | +| [project_create](variables.tf#L83) | Provide values if project creation is needed, uses existing project if null. Parent is in 'folders/nnn' or 'organizations/nnn' format. | object({…}) | | null | +| [region](variables.tf#L97) | Reagion where instances will be deployed. | string | | "europe-west1" | +| [service_encryption_keys](variables.tf#L103) | Cloud KMS keys to use to encrypt resources. Provide a key for each reagion in use. | map(string) | | null | ## Outputs | name | description | sensitive | |---|---|:---:| -| [composer_airflow_uri](outputs.tf#L22) | The URI of the Apache Airflow Web UI hosted within the Cloud Composer environment.. | | -| [composer_dag_gcs](outputs.tf#L17) | The Cloud Storage prefix of the DAGs for the Cloud Composer environment. | | +| [composer_airflow_uri](outputs.tf#L17) | The URI of the Apache Airflow Web UI hosted within the Cloud Composer environment.. | | +| [composer_dag_gcs](outputs.tf#L22) | The Cloud Storage prefix of the DAGs for the Cloud Composer environment. | | diff --git a/blueprints/data-solutions/composer-2/outputs.tf b/blueprints/data-solutions/composer-2/outputs.tf index a2943006e1..4e09a04985 100644 --- a/blueprints/data-solutions/composer-2/outputs.tf +++ b/blueprints/data-solutions/composer-2/outputs.tf @@ -14,12 +14,12 @@ * limitations under the License. */ -output "composer_dag_gcs" { - description = "The Cloud Storage prefix of the DAGs for the Cloud Composer environment." - value = google_composer_environment.env.config[0].dag_gcs_prefix -} - output "composer_airflow_uri" { description = "The URI of the Apache Airflow Web UI hosted within the Cloud Composer environment.." value = google_composer_environment.env.config[0].airflow_uri } + +output "composer_dag_gcs" { + description = "The Cloud Storage prefix of the DAGs for the Cloud Composer environment." + value = google_composer_environment.env.config[0].dag_gcs_prefix +} diff --git a/blueprints/data-solutions/data-platform-foundations/README.md b/blueprints/data-solutions/data-platform-foundations/README.md index d2b8855046..034bb32a72 100644 --- a/blueprints/data-solutions/data-platform-foundations/README.md +++ b/blueprints/data-solutions/data-platform-foundations/README.md @@ -251,11 +251,11 @@ You can find examples in the `[demo](./demo)` folder. | [folder_id](variables.tf#L53) | Folder to be used for the networking resources in folders/nnnn format. | string | ✓ | | | [organization_domain](variables.tf#L98) | Organization domain. | string | ✓ | | | [prefix](variables.tf#L103) | Unique prefix used for resource names. | string | ✓ | | -| [composer_config](variables.tf#L22) | Cloud Composer config. | object({…}) | | {…} | +| [composer_config](variables.tf#L22) | Cloud Composer config. | object({…}) | | {…} | | [data_catalog_tags](variables.tf#L36) | List of Data Catalog Policy tags to be created with optional IAM binging configuration in {tag => {ROLE => [MEMBERS]}} format. | map(map(list(string))) | | {…} | | [data_force_destroy](variables.tf#L47) | Flag to set 'force_destroy' on data services like BiguQery or Cloud Storage. | bool | | false | -| [groups](variables.tf#L64) | User groups. | map(string) | | {…} | -| [location](variables.tf#L58) | Location used for multi-regional resources. | string | | "eu" | +| [groups](variables.tf#L58) | User groups. | map(string) | | {…} | +| [location](variables.tf#L68) | Location used for multi-regional resources. | string | | "eu" | | [network_config](variables.tf#L74) | Shared VPC network configurations to use. If null networks will be created in projects with preconfigured values. | object({…}) | | null | | [project_services](variables.tf#L108) | List of core services enabled on all projects. | list(string) | | […] | | [project_suffix](variables.tf#L119) | Suffix used only for project ids. | string | | null | diff --git a/blueprints/data-solutions/data-platform-foundations/variables.tf b/blueprints/data-solutions/data-platform-foundations/variables.tf index dc59de2931..adf3c7e49e 100644 --- a/blueprints/data-solutions/data-platform-foundations/variables.tf +++ b/blueprints/data-solutions/data-platform-foundations/variables.tf @@ -55,12 +55,6 @@ variable "folder_id" { type = string } -variable "location" { - description = "Location used for multi-regional resources." - type = string - default = "eu" -} - variable "groups" { description = "User groups." type = map(string) @@ -71,6 +65,12 @@ variable "groups" { } } +variable "location" { + description = "Location used for multi-regional resources." + type = string + default = "eu" +} + variable "network_config" { description = "Shared VPC network configurations to use. If null networks will be created in projects with preconfigured values." type = object({ diff --git a/blueprints/networking/nginx-reverse-proxy-cluster/README.md b/blueprints/networking/nginx-reverse-proxy-cluster/README.md index 9cacb4b045..c3101a1500 100644 --- a/blueprints/networking/nginx-reverse-proxy-cluster/README.md +++ b/blueprints/networking/nginx-reverse-proxy-cluster/README.md @@ -22,19 +22,19 @@ Repository and set the `ops_agent_image` to point to the image you built. | name | description | type | required | default | |---|---|:---:|:---:|:---:| | [autoscaling_metric](variables.tf#L31) | | object({…} | ✓ | | -| [project_name](variables.tf#L106) | Name of an existing project or of the new project | string | ✓ | | +| [project_name](variables.tf#L108) | Name of an existing project or of the new project | string | ✓ | | | [autoscaling](variables.tf#L17) | Autoscaling configuration for the instance group. | object({…}) | | {…} | -| [backends](variables.tf#L49) | Nginx locations configurations to proxy traffic to. | string | | "<<-EOT…EOT" | -| [cidrs](variables.tf#L59) | Subnet IP CIDR ranges. | map(string) | | {…} | -| [network](variables.tf#L67) | Network name. | string | | "reverse-proxy-vpc" | -| [network_create](variables.tf#L73) | Create network or use existing one. | bool | | true | -| [nginx_image](variables.tf#L79) | Nginx container image to use. | string | | "gcr.io/cloud-marketplace/google/nginx1:latest" | -| [ops_agent_image](variables.tf#L85) | Google Cloud Ops Agent container image to use. | string | | "gcr.io/sfans-hub-project-d647/ops-agent:latest" | -| [prefix](variables.tf#L91) | Prefix used for resources that need unique names. | string | | "" | -| [project_create](variables.tf#L97) | Parameters for the creation of the new project | object({…}) | | null | -| [region](variables.tf#L111) | Default region for resources. | string | | "europe-west4" | -| [subnetwork](variables.tf#L117) | Subnetwork name. | string | | "gce" | -| [tls](variables.tf#L123) | Also offer reverse proxying with TLS (self-signed certificate). | bool | | false | +| [backends](variables.tf#L49) | Nginx locations configurations to proxy traffic to. | string | | "<<-EOT…EOT" | +| [cidrs](variables.tf#L61) | Subnet IP CIDR ranges. | map(string) | | {…} | +| [network](variables.tf#L69) | Network name. | string | | "reverse-proxy-vpc" | +| [network_create](variables.tf#L75) | Create network or use existing one. | bool | | true | +| [nginx_image](variables.tf#L81) | Nginx container image to use. | string | | "gcr.io/cloud-marketplace/google/nginx1:latest" | +| [ops_agent_image](variables.tf#L87) | Google Cloud Ops Agent container image to use. | string | | "gcr.io/sfans-hub-project-d647/ops-agent:latest" | +| [prefix](variables.tf#L93) | Prefix used for resources that need unique names. | string | | "" | +| [project_create](variables.tf#L99) | Parameters for the creation of the new project | object({…}) | | null | +| [region](variables.tf#L113) | Default region for resources. | string | | "europe-west4" | +| [subnetwork](variables.tf#L119) | Subnetwork name. | string | | "gce" | +| [tls](variables.tf#L125) | Also offer reverse proxying with TLS (self-signed certificate). | bool | | false | ## Outputs diff --git a/blueprints/third-party-solutions/wordpress/cloudrun/README.md b/blueprints/third-party-solutions/wordpress/cloudrun/README.md index 10e5be70ca..ee1e2d909f 100644 --- a/blueprints/third-party-solutions/wordpress/cloudrun/README.md +++ b/blueprints/third-party-solutions/wordpress/cloudrun/README.md @@ -115,18 +115,19 @@ The above command will delete the associated resources so there will be no billa | name | description | type | required | default | |---|---|:---:|:---:|:---:| -| [project_id](variables.tf#L72) | Project id, references existing project if `project_create` is null. | string | ✓ | | -| [wordpress_image](variables.tf#L83) | Image to run with Cloud Run, starts with \"gcr.io\" | string | ✓ | | +| [project_id](variables.tf#L78) | Project id, references existing project if `project_create` is null. | string | ✓ | | +| [wordpress_image](variables.tf#L89) | Image to run with Cloud Run, starts with \"gcr.io\" | string | ✓ | | | [cloud_run_invoker](variables.tf#L18) | IAM member authorized to access the end-point (for example, 'user:YOUR_IAM_USER' for only you or 'allUsers' for everyone) | string | | "allUsers" | | [cloudsql_password](variables.tf#L24) | CloudSQL password (will be randomly generated by default) | string | | null | -| [create_connector](variables.tf#L30) | Should a VPC serverless connector be created or not | bool | | true | -| [ip_ranges](variables.tf#L37) | CIDR blocks: VPC serverless connector, Private Service Access(PSA) for CloudSQL, CloudSQL VPC | object({…}) | | {…} | -| [prefix](variables.tf#L51) | Unique prefix used for resource names. Not used for project if 'project_create' is null. | string | | "" | -| [principals](variables.tf#L57) | List of users to give rights to (CloudSQL admin, client and instanceUser, Logging admin, Service Account User and TokenCreator), eg 'user@domain.com'. | list(string) | | [] | -| [project_create](variables.tf#L63) | Provide values if project creation is needed, uses existing project if null. Parent is in 'folders/nnn' or 'organizations/nnn' format. | object({…}) | | null | -| [region](variables.tf#L77) | Region for the created resources | string | | "europe-west4" | +| [connector](variables.tf#L30) | Existing VPC serverless connector to use if not creating a new one | string | | null | +| [create_connector](variables.tf#L36) | Should a VPC serverless connector be created or not | bool | | true | +| [ip_ranges](variables.tf#L43) | CIDR blocks: VPC serverless connector, Private Service Access(PSA) for CloudSQL, CloudSQL VPC | object({…}) | | {…} | +| [prefix](variables.tf#L57) | Unique prefix used for resource names. Not used for project if 'project_create' is null. | string | | "" | +| [principals](variables.tf#L63) | List of users to give rights to (CloudSQL admin, client and instanceUser, Logging admin, Service Account User and TokenCreator), eg 'user@domain.com'. | list(string) | | [] | +| [project_create](variables.tf#L69) | Provide values if project creation is needed, uses existing project if null. Parent is in 'folders/nnn' or 'organizations/nnn' format. | object({…}) | | null | +| [region](variables.tf#L83) | Region for the created resources | string | | "europe-west4" | | [wordpress_password](variables.tf#L94) | Password for the Wordpress user (will be randomly generated by default) | string | | null | -| [wordpress_port](variables.tf#L88) | Port for the Wordpress image | number | | 8080 | +| [wordpress_port](variables.tf#L100) | Port for the Wordpress image | number | | 8080 | ## Outputs @@ -134,7 +135,7 @@ The above command will delete the associated resources so there will be no billa |---|---|:---:| | [cloud_run_service](outputs.tf#L17) | CloudRun service URL | ✓ | | [cloudsql_password](outputs.tf#L23) | CloudSQL password | ✓ | -| [wp_password](outputs.tf#L34) | Wordpress user password | ✓ | -| [wp_user](outputs.tf#L29) | Wordpress username | | +| [wp_password](outputs.tf#L29) | Wordpress user password | ✓ | +| [wp_user](outputs.tf#L35) | Wordpress username | | diff --git a/blueprints/third-party-solutions/wordpress/cloudrun/outputs.tf b/blueprints/third-party-solutions/wordpress/cloudrun/outputs.tf index 349e5b5664..3216f79ee0 100644 --- a/blueprints/third-party-solutions/wordpress/cloudrun/outputs.tf +++ b/blueprints/third-party-solutions/wordpress/cloudrun/outputs.tf @@ -26,13 +26,13 @@ output "cloudsql_password" { sensitive = true } -output "wp_user" { - description = "Wordpress username" - value = local.wp_user -} - output "wp_password" { description = "Wordpress user password" value = local.wp_pass sensitive = true } + +output "wp_user" { + description = "Wordpress username" + value = local.wp_user +} diff --git a/blueprints/third-party-solutions/wordpress/cloudrun/variables.tf b/blueprints/third-party-solutions/wordpress/cloudrun/variables.tf index eaa2543b9f..426ffe76a3 100644 --- a/blueprints/third-party-solutions/wordpress/cloudrun/variables.tf +++ b/blueprints/third-party-solutions/wordpress/cloudrun/variables.tf @@ -91,14 +91,14 @@ variable "wordpress_image" { description = "Image to run with Cloud Run, starts with \"gcr.io\"" } +variable "wordpress_password" { + type = string + description = "Password for the Wordpress user (will be randomly generated by default)" + default = null +} + variable "wordpress_port" { type = number description = "Port for the Wordpress image" default = 8080 } - -variable "wordpress_password" { - type = string - description = "Password for the Wordpress user (will be randomly generated by default)" - default = null -} \ No newline at end of file diff --git a/tools/check_documentation.py b/tools/check_documentation.py index 2a6dd8ecf7..fbf4e59da7 100755 --- a/tools/check_documentation.py +++ b/tools/check_documentation.py @@ -49,15 +49,32 @@ def _check_dir(dir_name, exclude_files=None, files=False, show_extra=False): try: new_doc = tfdoc.create_doc(readme_path.parent, files, show_extra, exclude_files, readme) + variables = [v.name for v in new_doc.variables] + outputs = [v.name for v in new_doc.outputs] except SystemExit: state = state.SKIP else: - if new_doc == result['doc']: + if new_doc.content == result['doc']: state = State.OK + elif variables != sorted(variables): + state = state.FAIL + diff = "\n".join([ + f'----- {mod_name} variables -----', + f'variables should be in this order: ', + ', '.join(sorted(variables)), + ]) + elif outputs != sorted(outputs): + state = state.FAIL + diff = "\n".join([ + f'----- {mod_name} outputs -----', + f'outputs should be in this order: ', + ', '.join(sorted(outputs)), + ]) else: state = State.FAIL header = f'----- {mod_name} diff -----\n' - ndiff = difflib.ndiff(result['doc'].split('\n'), new_doc.split('\n')) + ndiff = difflib.ndiff(result['doc'].split('\n'), + new_doc.content.split('\n')) diff = '\n'.join([header] + list(ndiff)) yield mod_name, state, diff @@ -73,7 +90,7 @@ def main(dirs, exclude_file=None, files=False, show_diffs=False, 'Cycle through modules and ensure READMEs are up-to-date.' print(f'files: {files}, extra: {show_extra}, diffs: {show_diffs}\n') errors = [] - state_labels = {State.FAIL: '✗', State.OK: '✓', State.SKIP: '?'} + state_labels = {State.FAIL: '✗', State.OK: '✓', State.SKIP: ' '} for dir_name in dirs: print(f'----- {dir_name} -----') for mod_name, state, diff in _check_dir(dir_name, exclude_file, files, diff --git a/tools/tfdoc.py b/tools/tfdoc.py index d7771fb58f..d06dedb9e3 100755 --- a/tools/tfdoc.py +++ b/tools/tfdoc.py @@ -99,13 +99,13 @@ VAR_RE_TYPE = re.compile(r'([\(\{\}\)])') VAR_TEMPLATE = ('default', 'description', 'type', 'nullable') +Document = collections.namedtuple('Document', 'content files variables outputs') File = collections.namedtuple('File', 'name description modules resources') Output = collections.namedtuple( 'Output', 'name description sensitive consumers file line') Variable = collections.namedtuple( 'Variable', 'name description type default required nullable source file line') - # parsing functions @@ -247,7 +247,7 @@ def format_doc(outputs, variables, files, show_extra=False): def format_files(items): 'Format files table.' - items.sort(key=lambda i: i.name) + items = sorted(items, key=lambda i: i.name) num_modules = sum(len(i.modules) for i in items) num_resources = sum(len(i.resources) for i in items) yield '| name | description |{}{}'.format( @@ -271,7 +271,7 @@ def format_outputs(items, show_extra=True): 'Format outputs table.' if not items: return - items.sort(key=lambda i: i.name) + items = sorted(items, key=lambda i: i.name) yield '| name | description | sensitive |' + (' consumers |' if show_extra else '') yield '|---|---|:---:|' + ('---|' if show_extra else '') @@ -289,8 +289,7 @@ def format_variables(items, show_extra=True): 'Format variables table.' if not items: return - items.sort(key=lambda i: i.name) - items.sort(key=lambda i: i.required, reverse=True) + items = sorted(items, key=lambda i: (not i.required, i.name)) yield '| name | description | type | required | default |' + ( ' producer |' if show_extra else '') yield '|---|---|:---:|:---:|:---:|' + (':---:|' if show_extra else '') @@ -358,7 +357,8 @@ def create_doc(module_path, files=False, show_extra=False, exclude_files=None, mod_outputs = list(parse_outputs(module_path, exclude_files)) except (IOError, OSError) as e: raise SystemExit(e) - return format_doc(mod_outputs, mod_variables, mod_files, show_extra) + doc = format_doc(mod_outputs, mod_variables, mod_files, show_extra) + return Document(doc, mod_files, mod_variables, mod_outputs) def get_readme(readme_path): @@ -402,7 +402,7 @@ def main(module_path=None, exclude_file=None, files=False, replace=True, readme = get_readme(readme_path) doc = create_doc(module_path, files, show_extra, exclude_file, readme) if replace: - replace_doc(readme_path, doc, readme) + replace_doc(readme_path, doc.content, readme) else: print(doc)