Merge pull request #18 from ukgovdatascience/refactor_hooks

Refactor cookiecutter hooks and tests

.coveragerc

@@ -2,3 +2,6 @@
 source =
     ./hooks
     ./{{ cookiecutter.repo_name }}/src
+[report]
+exclude_lines =
+    if __name__ == .__main__.:

.gitignore

@@ -856,6 +856,9 @@ MigrationBackup/
 
 ### govcookiecutter ###
 
+# Ignore the `docs/reference/api` folder
+docs/reference/api/*
+
 # Ignore `example` folder, but not its `README.md`
 example/*
 !example/README.md

CONTRIBUTING.md

@@ -11,6 +11,7 @@ still have questions, please [contact us][email] and we'd be happy to help!
   - [Markdown](#markdown)
 - [Testing](#testing)
 - [Documentation](#documentation)
+- [HM Government department frameworks](#hm-government-department-frameworks)
 
 ## Code of Conduct
 
@@ -100,13 +101,20 @@ unless it's more appropriate to store it elsewhere, like this file.
 Further information on how to write Sphinx documentation, and how to build it into a searchable website can be found
 [here][docs-write-sphinx-documentation].
 
+## HM Government department frameworks
+
+HM Government frameworks are stored in the `.govcookiecutter/department_frameworks` folder. If you would like to add
+your own department's framework, follow the [instructions][docs-govcookiecutter-frameworks] in the `README.md` file in
+that folder.
+
 ---
 
 <a name="footnote-1">[1]</a>: Only secrets of specific patterns are detected by the pre-commit hooks. See
 [here][docs-pre-commit-hooks-secrets-definition] for further details.
 
 [code-of-conduct]: ./CODE_OF_CONDUCT.md
 [coverage]: https://coverage.readthedocs.io/
+[docs-govcookiecutter-frameworks]: ./%7B%7B%20cookiecutter.repo_name%20%7D%7D/.govcookiecutter/department_frameworks/README.md
 [docs-pre-commit-hooks]: ./%7B%7B%20cookiecutter.repo_name%20%7D%7D/docs/contributor_guide/pre_commit_hooks.md
 [docs-pre-commit-hooks-secrets-definition]: ./%7B%7B%20cookiecutter.repo_name%20%7D%7D/docs/contributor_guide/pre_commit_hooks.md#definition-of-a-secret-according-to-detect-secrets
 [docs-updating-gitignore]: ./%7B%7B%20cookiecutter.repo_name%20%7D%7D/docs/contributor_guide/updating_gitignore.md
@@ -116,6 +124,7 @@ Further information on how to write Sphinx documentation, and how to build it in
 [gds-way-git]: https://gds-way.cloudapps.digital/standards/source-code.html
 [gds-way-python]: https://gds-way.cloudapps.digital/manuals/programming-languages/python/python.html#python-style-guide
 [govcookiecutter]: https://github.com/ukgovdatascience/govcookiecutter
+[govcookiecutter-frameworks]: ./%7B%7B%20cookiecutter.repo_name%20%7D%7D/.govcookiecutter/department_frameworks
 [myst]: https://myst-parser.readthedocs.io/
 [pre-commit]: https://pre-commit.com/
 [pytest]: https://docs.pytest.org/

Makefile

@@ -1,4 +1,7 @@
 .PHONY:
+	coverage
+	coverage_html
+	coverage_xml
 	docs
 	docs_check_external_links
 	example
@@ -42,6 +45,18 @@ example: prepare_example_folder requirements
 example_with_options: prepare_example_folder requirements
 	python3 -m cookiecutter . -o ./example
 
+## Run code coverage
+coverage: requirements
+	coverage run -m pytest
+
+## Run code coverage, and produce a HTML output
+coverage_html: coverage
+	coverage html
+
+## Run code coverage, and produce an XML output
+coverage_xml: coverage
+	coverage xml
+
 ## Get help on all make commands; referenced from https://github.com/drivendata/cookiecutter-data-science
 help:
 	@echo "$$(tput bold)Available rules:$$(tput sgr0)"

cookiecutter.json

@@ -5,7 +5,7 @@
   "repository_hosting_platform": ["GitHub", "GitLab"],
   "organisation_name": "Your GitHub/GitLab organisation name",
   "contact_email": "[email protected]",
-  "departmental_aqa_framework": ["GDS"],
+  "departmental_framework": ["GDS", "N/A"],
 
   "project_name": "Your new project name",
   "repo_name": "{{ cookiecutter.project_name.lower().replace(' ', '-') }}",

docs/conf.py

@@ -1,4 +1,4 @@
-# govcookiecutter documentation build configuration file
+# `govcookiecutter` documentation build configuration file
 #
 # This file is execfile()d with the current directory set to its containing dir.
 #
@@ -9,8 +9,9 @@
 # If extensions (or modules to document with autodoc) are in another directory, add these directories to sys.path here.
 # If the directory is relative to the documentation root, use os.path.abspath to make it absolute, like shown here.
 
-# import sys
-# sys.path.insert(0, os.path.abspath("."))
+import os
+import sys
+sys.path.insert(0, os.path.abspath(".."))
 
 # -- General configuration ------------------------------------------------
 

docs/guides/contributing_guide.md

@@ -6,6 +6,7 @@ This is the contributor guide for the `govcookiecutter` project.
 :maxdepth: 2
 ./CODE_OF_CONDUCT.md
 ./CONTRIBUTING.md
+./{{ cookiecutter.repo_name }}/.govcookiecutter/department_frameworks/README.md
 ./{{ cookiecutter.repo_name }}/docs/contributor_guide/pre_commit_hooks.md
 ./{{ cookiecutter.repo_name }}/docs/contributor_guide/updating_gitignore.md
 ./{{ cookiecutter.repo_name }}/docs/contributor_guide/writing_sphinx_documentation.md

docs/guides/{{ cookiecutter.repo_name }}/.govcookiecutter/department_frameworks/README.md

@@ -0,0 +1,2 @@
+```{include} ../../../../../{{ cookiecutter.repo_name }}/.govcookiecutter/department_frameworks/README.md
+```

docs/index.md

@@ -7,4 +7,5 @@ Here is the documentation for `govcookiecutter`.
 ./guides/README.md
 ./guides/contributing_guide.md
 ./structure/README.md
+./reference/README.md
 ```

docs/reference/README.md

@@ -0,0 +1,10 @@
+# `hooks` API reference
+
+This page gives an overview of all public `hooks` objects, functions and methods. All classes and functions exposed
+in `hooks.*` namespace are public.
+
+```{toctree}
+:maxdepth: 2
+./pre_gen_project.md
+./post_gen_project.md
+```

docs/reference/post_gen_project.md

@@ -0,0 +1,20 @@
+# Post-cookiecutter generation hooks
+
+These `hooks` package functions are executed after the project structure has been generated.
+
+<!-- Functions should be referenced in the `hooks.__init__.py` -->
+```{eval-rst}
+.. currentmodule:: hooks
+```
+
+## HM Government department framework functions
+
+```{eval-rst}
+.. autosummary::
+    :toctree: api/
+
+    remove_folder
+    set_aqa_framework
+    set_request_template
+
+```

docs/reference/pre_gen_project.md

@@ -0,0 +1,18 @@
+# Pre-cookiecutter generation hooks
+
+These `hooks` package functions are executed before the project structure has been generated.
+
+<!-- Functions should be referenced in the `hooks.__init__.py` -->
+```{eval-rst}
+.. currentmodule:: hooks
+```
+
+## Cookiecutter input validation
+
+```{eval-rst}
+.. autosummary::
+    :toctree: api/
+
+    check_valid_email_address_format
+
+```

hooks/__init__.py

@@ -0,0 +1,3 @@
+from hooks.post_gen_project import remove_folder, set_aqa_framework, set_request_template
+from hooks.pre_gen_project import check_valid_email_address_format
+__all__ = ("check_valid_email_address_format", "remove_folder", "set_aqa_framework", "set_request_template", )

hooks/post_gen_project.py

@@ -1,83 +1,101 @@
+from pathlib import Path
 from shutil import rmtree
-import os
+from typing import Union
 
-# Define the folder path to the 'docs/aqa_frameworks' folder, and the future `docs/aqa` folder
-PATH_DOCS_AQA_FRAMEWORKS = os.path.join("docs", "aqa_frameworks")
-PATH_DOCS_AQA = os.path.join("docs", "aqa")
 
-# Define the folder path to the `docs/pull_merge_request_templates` folder
-PATH_PR_MR_DEPT_TEMPLATES = os.path.join("docs", "pull_merge_request_templates")
+def remove_folder(folder: Union[Path, str]) -> None:
+    """Remove a folder.
 
-# Define the folder path where pull/merge requests should exist as values in a dictionary, where the keys are the
-# remote hosts
-PATH_PR_MR_TEMPLATE = {
-    "GitHub": [os.path.join(".github"), "pull_request_template.md"],
-    "GitLab": [os.path.join(".gitlab", "merge_request_templates"), "{{ cookiecutter.project_name }}.md"]
-}
+    Args:
+        folder: A folder path to be removed.
+
+    Returns:
+        None - removes the `folder` folder.
+
+    """
+    rmtree(folder)
 
 
-def select_dept_aqa_framework(user_option: str, default_option: str = "GDS") -> None:
-    """Create analytical quality assurance (AQA) documents for a specific HM Government department.
+def set_aqa_framework(dir_department_framework_aqa: Union[Path, str],
+                      dir_cookiecutter_docs_aqa: Union[Path, str]) -> None:
+    """Set a specific HM Government department analytical quality assurance (AQA) framework.
 
     Args:
-        user_option (str): User option that defines a HM Government departmental AQA framework to use.
-        default_option (str): Default: GDS. Default option if user_option is not an existing framework.
+        dir_department_framework_aqa: A folder path that contains a specific HM Government department AQA framework.
+        dir_cookiecutter_docs_aqa: A folder path within the outputted project structure, where the contents of
+            `dir_department_framework_aqa` will reside.
 
     Returns:
-        A new folder in docs/aqa containing the selected HM Government departmental AQA framework.
+        The department-specific AQA framework in the outputted project structure's `dir_cookiecutter_docs_aqa` folder.
 
     """
 
-    # Get all the directories in `PATH_DOCS_AQA_FRAMEWORKS`
-    all_folders = [d for d in os.listdir(PATH_DOCS_AQA_FRAMEWORKS) if os.path.isdir(d)]
+    # Remove the default `docs/aqa` folder, and its contents. Then recursively create a new folder path to here
+    remove_folder(dir_cookiecutter_docs_aqa)
 
-    # Select the correct folder; use `default_option` if `user_option` is not a valid sub-folder in `all_folders`
-    selected_folder = user_option if user_option in all_folders else default_option
+    # Copy the relevant HM Government departmental AQA framework to the `docs/aqa` folder
+    _ = Path(dir_department_framework_aqa).rename(dir_cookiecutter_docs_aqa)
 
-    # Copy the relevant HM Government departmental AQA framework to the `docs/aqa` folder, using `default_option` if
-    # `user_option` is not a valid sub-folder in `PATH_DOCS_AQA_FRAMEWORKS`
-    os.rename(os.path.join(PATH_DOCS_AQA_FRAMEWORKS, selected_folder), PATH_DOCS_AQA)
 
-    # Remove all framework folders now we have the correct AQA framework
-    rmtree(PATH_DOCS_AQA_FRAMEWORKS)
+def set_request_template(path_department_framework_request_template: Union[Path, str],
+                         dir_govcookiecutter: Union[Path, str], repository_hosting_platform: str) -> None:
+    """Set a pull or merge request template in the outputted project structure for a specific HM Government department.
 
-
-def select_pull_merge_request_template(user_option: str, repo_host: str, default_option: str = "GDS") -> None:
-    """Select a pull/merge request template depending on HM Government department, and repository remote host.
+    A pull request template is created if the user chooses GitHub as their repository hosting platform. A merge request
+    template is created if they choose GitLab instead.
 
     Args:
-        user_option (str): User option that defines a HM Government departmental pull/merge request template to use.
-        repo_host (str): Repository hosting name.
-        default_option (str): Default: GDS. Default option if user_option is not an existing pull/merge request
-            template.
+        path_department_framework_request_template: A file path to the specific HM Government department pull or merge
+            request template.
+        dir_govcookiecutter: A folder path to the outputted `govcookiecutter` template.
+        repository_hosting_platform: The repository hosting platform. Must be one of "GitHub" or "GitLab" (case
+            insensitive).
 
     Returns:
-        A pull/merge request template in the correct location for the selected repo_host.
+        A department-specific pull or merge request template in the correct location, depending on their choice of
+        GitHub or GitLab as the repository hosting platform. If neither GitHub or GitLab are chosen, a `ValueError` is
+        raised.
 
     """
 
-    # Get all Markdown files in `PATH_PR_MR_DEPT_TEMPLATES`
-    md_files = [os.path.splitext(f)[0] for f in os.listdir(PATH_PR_MR_DEPT_TEMPLATES) if f.endswith(".md")]
+    # Define the file path where the request template will be moved, which is dependent on the repository hosting
+    # platform. If the `dir_request_template` is not initially one of `.github` or `.gitlab`, raise a `ValueError`
+    if repository_hosting_platform.lower() == "github":
+        path_request_template = Path(dir_govcookiecutter).joinpath(f".{repository_hosting_platform.lower()}",
+                                                                   "pull_request_template.md")
+    elif repository_hosting_platform.lower() == "gitlab":
+        path_request_template = Path(dir_govcookiecutter).joinpath(f".{repository_hosting_platform.lower()}",
+                                                                   "merge_request_templates",
+                                                                   "{{ cookiecutter.project_name }}.md")
+    else:
+        raise ValueError("`repository_hosting_platform` must be one of `GitHub` or `GitLab`: "
+                         f"{repository_hosting_platform}")
+
+    # Recursively create all the directories to the parent directory of `path_request_template` if they do not already
+    # exist
+    if not path_request_template.parent.is_dir():
+        path_request_template.parent.mkdir(parents=True, exist_ok=True)
 
-    # Determine the selected file; fallback to `default_option` if `user_option` is not in `md_files`
-    selected_md_file = f"{user_option if user_option in md_files else default_option}.md"
+    # Move the `path_department_framework_request_template` to `path_request_template`
+    _ = Path(path_department_framework_request_template).rename(path_request_template)
 
-    # Create a directory for the new location
-    if not os.path.isdir(PATH_PR_MR_TEMPLATE[repo_host][0]):
-        os.makedirs(PATH_PR_MR_TEMPLATE[repo_host][0], exist_ok=True)
 
-    # Move the `selected_md_file` to the correct location
-    os.rename(os.path.join(PATH_PR_MR_DEPT_TEMPLATES, selected_md_file), os.path.join(*PATH_PR_MR_TEMPLATE[repo_host]))
+if __name__ == "__main__":
 
-    # Remove all pull/merge request templates now that we have the correct one
-    rmtree(PATH_PR_MR_DEPT_TEMPLATES)
+    # Define the folder path to `.govcookiecutter`
+    DIR_GOVCOOKIECUTTER = Path(".govcookiecutter")
 
+    # Check `{{ cookiecutter.departmental_framework }}` is not `N/A`
+    if "{{ cookiecutter.departmental_framework }}" != "N/A":
 
-if __name__ == "__main__":
+        # Define the folder path to the specific department framework of interest in the `department_frameworks` folder
+        DIR_DEPARTMENT_FRAMEWORKS = DIR_GOVCOOKIECUTTER.joinpath("department_frameworks",
+                                                                 "{{ cookiecutter.departmental_framework }}")
 
-    # Select the appropriate AQA framework
-    select_dept_aqa_framework("{{ cookiecutter.departmental_aqa_framework }}")
+        # Transfer the `aqa` folder, and the pull/merge request templates to the correct folder paths
+        set_aqa_framework(DIR_DEPARTMENT_FRAMEWORKS.joinpath("aqa"), Path("docs").joinpath("aqa"))
+        set_request_template(DIR_DEPARTMENT_FRAMEWORKS.joinpath("request_template.md"), Path.cwd(),
+                             "{{ cookiecutter.repository_hosting_platform }}")
 
-    # Select the appropriate pull/merge request template
-    select_pull_merge_request_template("{{ cookiecutter.departmental_aqa_framework }}",
-                                       "{{ cookiecutter.repository_hosting_platform }}")
+    # Remove `DIR_GOVCOOKIECUTTER`
+    remove_folder(DIR_GOVCOOKIECUTTER)

hooks/pre_gen_project.py

@@ -0,0 +1,31 @@
+import re
+
+# Regular expression to check for a valid email address — based on the HTML5 standard
+# (https://html.spec.whatwg.org/multipage/input.html#valid-e-mail-address). This is more restrictive than the RFC
+# standard; see the comments in this SO answer for further information: https://stackoverflow.com/a/201378
+REGEX_EMAIL_ADDRESS = r"^[a-zA-Z0-9.!#$%&'*+\/=?^_`{|}~-]+@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?" \
+                      r"(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*$"
+
+
+def check_valid_email_address_format(email: str) -> None:
+    """Check that an email address is of a valid format using regular expressions.
+
+    Uses the HTML5 standard for email address format. The regular expression pattern is available `here`_.
+
+    Args:
+        email: An email address to validate.
+
+    Returns:
+        None - raises an `AssertionError` if `email` is not a valid email address format.
+
+    .. _here:
+        https://html.spec.whatwg.org/multipage/input.html#valid-e-mail-address
+
+    """
+    assert bool(re.fullmatch(REGEX_EMAIL_ADDRESS, email)), f"Invalid email address supplied: {email}"
+
+
+if __name__ == "__main__":
+
+    # Check the format of the contact email address supplied is a valid one
+    check_valid_email_address_format("{{ cookiecutter.contact_email }}")

pytest.ini

@@ -3,6 +3,7 @@ addopts =
     -vv
     --doctest-modules
     --ignore="./docs/"
+    --ignore="./example/"
     --ignore="./\{\{\ cookiecutter.repo_name\ \}\}/docs/"
 doctest_optionflags = NORMALIZE_WHITESPACE
 testpaths =

requirements.txt

@@ -4,4 +4,6 @@ detect-secrets==1.0.3
 myst-parser
 pre-commit
 pytest
+pytest-mock
+pytest-xdist
 Sphinx