Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Extend tfdoc to generate TOCs #1538

Merged
merged 9 commits into from
Jul 28, 2023
Merged
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
Extend tfdoc to generate TOCs
juliocc committed Jul 28, 2023
commit 4f123ccc74f35ef6bdf1ae1bc5f66d430714b6ae
14 changes: 11 additions & 3 deletions modules/artifact-registry/README.md
Original file line number Diff line number Diff line change
@@ -2,6 +2,14 @@

This module simplifies the creation of repositories using Google Cloud Artifact Registry.

<!-- BEGIN TOC -->
- [Standard Repository](#standard-repository)
- [Remote and Virtual Repositories](#remote-and-virtual-repositories)
- [itional Docker and Maven Options](#itional-docker-and-maven-options)
- [Variables](#variables)
- [Outputs](#outputs)
<!-- END TOC -->

## Standard Repository

```hcl
@@ -58,12 +66,12 @@ module "registry-virtual" {
}
}

# tftest modules=3 resources=3 inventory=remote-virtual.yaml
# tst modules=3 resources=3 inventory=remote-virtual.yaml
```

## Additional Docker and Maven Options
## itional Docker and Maven Options

```hcl
```

module "registry-docker" {
source = "./fabric/modules/artifact-registry"
20 changes: 13 additions & 7 deletions modules/organization/README.md
Original file line number Diff line number Diff line change
@@ -10,20 +10,26 @@ This module allows managing several organization properties:

To manage organization policies, the `orgpolicy.googleapis.com` service should be enabled in the quota project.

## Features

## TOC
<!-- BEGIN TOC -->
- [TOC](#toc)
- [Example](#example)
- [IAM](#iam)
- [Organization Policies](#organization-policies)
- [Factory](#organization-policy-factory)
- [Custom Constraints](#organization-policy-custom-constraints)
- [Custom Constraints Factory](#organization-policy-custom-constraints-factory)
- [Organization Policy Factory](#organization-policy-factory)
- [Organization Policy Custom Constraints](#organization-policy-custom-constraints)
- [Organization Policy Custom Constraints Factory](#organization-policy-custom-constraints-factory)
- [Hierarchical Firewall Policies](#hierarchical-firewall-policies)
- [Directly Defined](#directly-defined-firewall-policies)
- [Factory](#firewall-policy-factory)
- [Directly Defined Firewall Policies](#directly-defined-firewall-policies)
- [Firewall Policy Factory](#firewall-policy-factory)
- [Log Sinks](#log-sinks)
- [Data Access Logs](#data-access-logs)
- [Custom Roles](#custom-roles)
- [Tags](#tags)
- [Files](#files)
- [Variables](#variables)
- [Outputs](#outputs)
<!-- END TOC -->

## Example

23 changes: 19 additions & 4 deletions tools/check_documentation.py
Original file line number Diff line number Diff line change
@@ -35,6 +35,7 @@ class State(enum.IntEnum):
SKIP = enum.auto()
OK = enum.auto()
FAIL_STALE_README = enum.auto()
FAIL_STALE_TOC = enum.auto()
FAIL_UNSORTED_VARS = enum.auto()
FAIL_UNSORTED_OUTPUTS = enum.auto()
FAIL_VARIABLE_PERIOD = enum.auto()
@@ -52,6 +53,7 @@ def label(self):
State.SKIP: ' ',
State.OK: '✓ ',
State.FAIL_STALE_README: '✗R',
State.FAIL_STALE_TOC: '✗T',
State.FAIL_UNSORTED_VARS: 'SV',
State.FAIL_UNSORTED_OUTPUTS: 'SO',
State.FAIL_VARIABLE_PERIOD: '.V',
@@ -71,8 +73,9 @@ def _check_dir(dir_name, exclude_files=None, files=False, show_extra=False):
diff = None
readme = readme_path.read_text()
mod_name = str(readme_path.relative_to(dir_path).parent)
result = tfdoc.get_doc(readme)
if not result:
doc_result = tfdoc.get_doc(readme)
toc_result = tfdoc.get_toc(readme)
if not doc_result:
state = State.SKIP
else:
try:
@@ -82,18 +85,30 @@ def _check_dir(dir_name, exclude_files=None, files=False, show_extra=False):
newouts = new_doc.outputs
variables = [v.name for v in newvars if v.file.endswith('variables.tf')]
outputs = [o.name for o in newouts if o.file.endswith('outputs.tf')]

new_toc = tfdoc.create_toc(readme)

except SystemExit:
state = state.SKIP
else:
state = State.OK

if new_doc.content != result['doc']:
if new_doc.content.strip() != doc_result['doc'].strip():
state = State.FAIL_STALE_README
header = f'----- {mod_name} diff -----\n'
ndiff = difflib.ndiff(result['doc'].split('\n'),
ndiff = difflib.ndiff(doc_result['doc'].split('\n'),
new_doc.content.split('\n'))
diff = '\n'.join([header] + list(ndiff))

if new_toc.strip() != toc_result['toc'].strip():
print(f"=====new\n{new_toc}")
print(f"=====result\n{toc_result['toc']}")
state = State.FAIL_STALE_TOC
header = f'----- {mod_name} diff -----\n'
ndiff = difflib.ndiff(toc_result['toc'].split('\n'),
new_toc.split('\n'))
diff = '\n'.join([header] + list(ndiff))

elif empty := [v.name for v in newvars if not v.description]:
state = state.FAIL_VARIABLE_DESCRIPTION
diff = "\n".join([
69 changes: 59 additions & 10 deletions tools/tfdoc.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
#!/usr/bin/env python3

# Copyright 2022 Google LLC
# Copyright 2023 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -46,6 +46,7 @@
import urllib.parse

import click
import marko

__version__ = '2.1.0'

@@ -80,6 +81,8 @@
''')
OUT_TEMPLATE = ('description', 'value', 'sensitive')
TAG_RE = re.compile(r'(?sm)^\s*#\stfdoc:([^:]+:\S+)\s+(.*?)\s*$')
TOC_BEGIN = '<!-- BEGIN TOC -->'
TOC_END = '<!-- END TOC -->'
UNESCAPED = string.digits + string.ascii_letters + ' .,;:_-'
VAR_ENUM = enum.Enum('V', 'OPEN ATTR ATTR_DATA SKIP CLOSE COMMENT TXT')
VAR_RE = re.compile(r'''(?smx)
@@ -322,17 +325,40 @@ def format_variables(items, show_extra=True):
yield format


def create_toc(readme):
doc = marko.parse(readme)
lines = []
headings = [x for x in doc.children if x.get_type() == 'Heading']
for h in headings[1:]:
title = h.children[0].children
slug = title.lower().strip()
slug = re.sub('[^\w\s-]', '', slug)
slug = re.sub('[-\s]+', '-', slug)
link = f'- [{title}](#{slug})'
indent = ' ' * (h.level - 2)
lines.append(f'{indent}{link}')
return "\n".join(lines)


# replace functions


def get_doc(readme):
'Check if README file is marked, and return current doc.'
m = re.search('(?sm)%s\n(.*)\n%s' % (MARK_BEGIN, MARK_END), readme)
m = re.search("(?sm)%s(.*)%s" % (MARK_BEGIN, MARK_END), readme)
juliocc marked this conversation as resolved.
Show resolved Hide resolved
if not m:
return
return {'doc': m.group(1), 'start': m.start(), 'end': m.end()}


def get_toc(readme):
'Check if README file is marked, and return current doc.'
t = re.search("(?sm)%s(.*)%s" % (TOC_BEGIN, TOC_END), readme)
if not t:
return
return {'toc': t.group(1), 'start': t.start(), 'end': t.end()}


def get_doc_opts(readme):
'Check if README file is setting options via a mark, and return options.'
m = MARK_OPTS_RE.search(readme)
@@ -373,22 +399,41 @@ def get_readme(readme_path):
raise SystemExit(f'Error opening README {readme_path}: {e}')


def replace_doc(readme_path, doc, readme=None):
def render_doc(readme, doc):
'Replace document in module\'s README.md file.'
readme = readme or get_readme(readme_path)
result = get_doc(readme)
if not result:
raise SystemExit(f'Mark not found in README {readme_path}')
juliocc marked this conversation as resolved.
Show resolved Hide resolved
raise SystemExit(f'Mark not found in README {readme}')
if doc == result['doc']:
return
return readme
try:
open(readme_path, 'w').write('\n'.join([
return '\n'.join([
readme[:result['start']].rstrip(),
MARK_BEGIN,
doc,
MARK_END,
readme[result['end']:].lstrip(),
]))
])
except (IOError, OSError) as e:
raise SystemExit(f'Error replacing README {readme_path}: {e}')


def render_toc(readme, toc):
'Replace document in module\'s README.md file.'
result = get_toc(readme)
if not result:
raise SystemExit(f'TOC not found in README {readme}')
if toc == result['toc']:
return readme
try:
return '\n'.join([
readme[:result['start']].rstrip(),
TOC_BEGIN,
toc,
TOC_END,
"",
readme[result['end']:].lstrip(),
])
except (IOError, OSError) as e:
raise SystemExit(f'Error replacing README {readme_path}: {e}')

@@ -405,10 +450,14 @@ def main(module_path=None, exclude_file=None, files=False, replace=True,
readme_path = os.path.join(module_path, 'README.md')
readme = get_readme(readme_path)
doc = create_doc(module_path, files, show_extra, exclude_file, readme)
toc = create_toc(readme)
tmp = render_doc(readme, doc.content)
final = render_toc(tmp, toc)
if replace:
replace_doc(readme_path, doc.content, readme)
with open(readme_path, 'w') as f:
f.write(final)
else:
print(doc)
print(final)


if __name__ == '__main__':