Merge branch 'main' into changelog

.github/workflows/govcookiecutter-build.yml

@@ -9,7 +9,7 @@ jobs:
     strategy:
       matrix:
         os: [ ubuntu-latest, macos-latest, windows-latest ]
-        python: [ 3.7, 3.8, 3.9, '3.10' ]
+        python: [ 3.8, 3.9, '3.10', 3.11 ]
 
     steps:
       - name: Checkout the revision

.github/workflows/govcookiecutter-template-build.yml

@@ -10,12 +10,11 @@ jobs:
       matrix:
         # TODO: Fix R pre-commit hook issues with Windows
         os: [ ubuntu-latest, macos-latest ]
-        python: [ 3.7, 3.8, 3.9, '3.10' ]
+        python: [ 3.8, 3.9, '3.10' ]
         R:
           - {using_R: No, version: N/A}
-          - {using_R: Yes, version: 4.0.4}
-          - {using_R: Yes, version: 4.0.5}
-          - {using_R: Yes, version: 4.1.0}
+          - {using_R: Yes, version: 4.1.3}
+          - {using_R: Yes, version: 4.2.2}
 
     steps:
       - name: Checkout the revision
@@ -33,7 +32,9 @@ jobs:
         if: ${{ matrix.R.using_r == 'Yes' }}
       - name: Install other ${{ matrix.os }} R dependencies
         run: |
+          sudo apt-get update
           sudo apt-get install libcurl4-openssl-dev
+          sudo apt-get install libgit2-dev
           sudo apt-get install libharfbuzz-dev
           sudo apt-get install libfribidi-dev
         if: ${{ matrix.os == 'ubuntu-latest' && matrix.R.using_R == 'Yes' }}
@@ -68,7 +69,7 @@ jobs:
             make docs
           elif [ "$RUNNER_OS" == "Windows" ]; then
             pip install -U pip setuptools
-            pip install -r requirements.txt
+            pip install -e .[dev]
             pre-commit install
             sphinx-build -b html ./docs ./docs/_build
           else

.gitignore

@@ -864,3 +864,9 @@ example/*
 
 # Ignore Sphinx documentation link checking folder
 docs/_linkcheck/
+
+# Commonly used data file extensions
+
+*.csv
+*.xls
+*.xlsx

.pre-commit-config.yaml

@@ -12,23 +12,25 @@ repos:
       - id: trailing-whitespace
         name: Check for trailing whitespaces (auto-fixes)
   - repo: https://github.com/pycqa/isort
-    rev: 5.8.0
+    rev: 5.12.0
     hooks:
       - id: isort
         name: isort - Sort Python imports (auto-fixes)
-        types: [ cython, pyi, python ]
         args: [ "--profile", "black", "--filter-files" ]
+        exclude: test_example_module.py|run_pipeline.py
   - repo: https://github.com/psf/black
     rev: 22.8.0 # Replace by any tag/version: https://github.com/psf/black/tags
     hooks:
       - id: black
         name: black - consistent Python code formatting (auto-fixes)
         language_version: python # Should be a command that runs python3.6+
+        exclude: test_example_module.py|run_pipeline.py
   - repo: https://github.com/PyCQA/flake8
     rev: 5.0.4
     hooks:
       - id: flake8
         name: flake8 - Python linting
+        exclude: test_example_module.py|run_pipeline.py
   - repo: https://github.com/Yelp/detect-secrets
     rev: v1.0.3
     hooks:

CONTRIBUTING.md

@@ -37,10 +37,7 @@ on pre-commit hooks see our documentation][docs-pre-commit-hooks].
 
 ### Git and GitHub
 
-We use Git to version control the source code. [Please read the GDS Way for details on
-Git best practice][gds-way-git]. This includes how to write good commit messages, use
-`git rebase` for local branches and `git merge --no-ff` for merges, as well as using
-`git push --force-with-lease` instead of `git push -f`.
+We use Git to version control the source code. [Please read the Quality assurance of code for analysis and research for details on Git best practice][duck-book-version-control]. This includes how to write good commit messages, how to branch correctly and solving merge conflicts.
 
 [If you want to modify the `.gitignore` files, see the template
 documentation][docs-updating-gitignore] for further details.
@@ -65,12 +62,11 @@ enforced in all cases, for example with long hyperlinks.
 ## Testing
 
 [Tests are written using the `pytest` framework][pytest], with its configuration in the
-`pyproject.toml` file. Note, only tests in the `tests`, and
-`{{ cookiecutter.repo_name }}/tests` folders folder are run. To run the tests, enter
+`pyproject.toml` file. Note, only the `tests` folder in the root direcrtory of this project are to run. To run the tests, enter
 the following command in your terminal:
 
 ```shell
-pytest
+pytest tests
 ```
 
 ### Code coverage
@@ -124,7 +120,7 @@ the `README.md` file in that folder.
 [docs-write-sphinx-documentation]: https://github.com/best-practice-and-impact/govcookiecutter/blob/main/%7B%7B%20cookiecutter.repo_name%20%7D%7D/docs/contributor_guide/writing_sphinx_documentation.md
 [email]: mailto:[email protected]
 [gds-way]: https://gds-way.cloudapps.digital/
-[gds-way-git]: https://gds-way.cloudapps.digital/standards/source-code.html
+[gds-way-git]: https://www.gov.uk/service-manual/technology/maintaining-version-control-in-coding
 [gds-way-python]: https://gds-way.cloudapps.digital/manuals/programming-languages/python/python.html#python-style-guide
 [govcookiecutter]: https://github.com/best-practice-and-impact/govcookiecutter
 [myst]: https://myst-parser.readthedocs.io/

README.md

@@ -4,7 +4,8 @@ A cookiecutter template for analytical, Python-, or Python and R-based projects
 His Majesty's Government, and wider public sector.
 
 This template helps to set up standardised project structures, and [includes security
-features using pre-commit hooks][docs-pre-commit].
+features using pre-commit hooks][docs-pre-commit]. This cookiecutter template also acts
+as an installable template (python projects only).
 
 It also provides an Agile, centralised, and lightweight analytical quality assurance
 (AQA) process. Pull or merge request templates are used to nudge users to complete this
@@ -22,13 +23,13 @@ terminal, navigate to the directory where you want your new repository to exist.
 run the following command for the latest stable release:
 
 ```shell
-cookiecutter https://github.com/best-practice-and-impact/govcookiecutter.git
+python -m cookiecutter https://github.com/best-practice-and-impact/govcookiecutter.git
 ```
 
 or for a specific branch, tag, or commit SHA `{SPECIFIC}`, run:
 
 ```shell
-cookiecutter https://github.com/best-practice-and-impact/govcookiecutter.git --checkout {SPECIFIC}
+python -m cookiecutter https://github.com/best-practice-and-impact/govcookiecutter.git --checkout {SPECIFIC}
 ```
 
 Follow the prompts; if you are asked to re-download `govcookiecutter`, input `yes`.
@@ -39,19 +40,24 @@ Once you've answered all the prompts, your project will be created. Then:
 
 1. Set up a Python virtual environment — [there are many ways to set up a virtual
   environment][pluralsight], so we'll let you decide what's best for you!
+
 2. In your terminal, navigate to your new project, and initialise Git
    ```shell
    git init
    ```
+
 3. Install the necessary packages using `pip` and the pre-commit hooks:
    ```shell
-   pip install -r requirements.txt
+   python -m pip install -U pip setuptools
+   python -m pip install -e .[dev]
    pre-commit install
    ```
+
    or use the `make` command:
    ```shell
-   make requirements
+   make install_dev
    ```
+
 4. Stage all your project files, and make your first commit
    ```shell
    git add .
@@ -133,7 +139,7 @@ guidelines](https://github.com/best-practice-and-impact/govcookiecutter/blob/mai
 ## Acknowledgements
 
 [This template is based off the DrivenData Cookiecutter Data Science
-project][drivendata]. Specifically, it uses similar `data` and `src` folder structures,
+project][drivendata]. Specifically, it uses a similar `src` folder structure,
 and a modified version of the `help` commands in the `Makefile`s.
 
 [aqua-book]: https://www.gov.uk/government/publications/the-aqua-book-guidance-on-producing-quality-analysis-for-government

cookiecutter.json

@@ -1,14 +1,12 @@
 {
-  "_extensions": ["jinja2_time.TimeExtension"],
-
   "organisation_name": "Your public sector organisation name, for example Government Digital Service",
   "repository_hosting_platform": ["GitHub", "GitLab"],
   "organisation_handle": "Your GitHub/GitLab organisation name, for example ukgovdatascience",
   "contact_email": "[email protected]",
   "organisational_framework": ["GDS", "N/A"],
 
   "project_name": "Your new project name",
-  "repo_name": "{{ cookiecutter.project_name.lower().replace(' ', '-') }}",
+  "repo_name": "{{ cookiecutter.project_name.lower().replace(' ', '_').replace('-', '_') }}",
   "overview": "Brief overview of your project.",
   "project_version": "0.0.1",
 

docs/accessibility.md

@@ -104,7 +104,7 @@ We plan to fix the accessibility issues in the content by the end of December 20
 
 This statement was prepared on 30 June 2021. It was last reviewed on 20 July 2021.
 
-[abilitynet]: https://mcmw.abilitynet.org.uk/
+[abilitynet]: https://abilitynet.org.uk/
 [accessibility-legislation]: https://www.legislation.gov.uk/uksi/2018/952/regulation/4/made
 [eass]: https://www.equalityadvisoryservice.com/
 [email]: mailto:[email protected]

pyproject.toml

@@ -4,6 +4,11 @@ source = [
     "./hooks",
     "./{{ cookiecutter.repo_name }}/src"
 ]
+omit = [
+    "./{{ cookiecutter.repo_name }}/src/{{ cookiecutter.repo_name.lower().replace(' ', '_').replace('-', '_') }}/run_pipeline.py"
+]
+
+[tool.coverage.omit]
 
 [tool.coverage.report]
 exclude_lines = [
@@ -25,6 +30,5 @@ addopts = [
 ]
 doctest_optionflags = "NORMALIZE_WHITESPACE"
 testpaths = [
-    "./tests",
-    "./{{ cookiecutter.repo_name }}/tests"
+    "./tests"
 ]

requirements.txt

@@ -1,9 +1,11 @@
+bandit
 cookiecutter
 coverage
 detect-secrets==1.0.3
 #pytest-cookies
 git+https://github.com/Jacobb164/pytest-cookies
 govuk-tech-docs-sphinx-theme
+jinja2-time
 myst-parser
 pre-commit
 pytest

tests/test_documentation_build.py

@@ -1,4 +1,5 @@
 from pathlib import Path
+
 from sphinx.cmd.build import main
 
 

tests/test_env.py

@@ -1,7 +1,8 @@
-from dotenv import dotenv_values
 from pathlib import Path
 from typing import Dict, List
 
+from dotenv import dotenv_values
+
 # Define a path to the `govcookiecutter` template directory, and its `.env` file
 DIR_TEMPLATE = Path("{{ cookiecutter.repo_name }}")
 PATH_TEMPLATE_ENV = DIR_TEMPLATE.joinpath(".env")
@@ -14,6 +15,45 @@
 EXCLUDE_SUB_DIR_IN_PARENTS_NAMES = [*EXCLUDE_ROOT_DIR_NAMES, "docs"]
 
 
+def remove_brackets_and_spaces(path_name: str) -> str:
+    """Removes spaces and brackets from a string.
+
+    Args:
+        path_name : A path name as a string.
+
+    Returns:
+        The path name with spaces and brackets removed.
+    """
+    if "{" in path_name or "}" in path_name or " " in path_name:
+        path_name = path_name.replace("{", "")
+        path_name = path_name.replace("}", "")
+        path_name = path_name.replace(" ", "")
+
+    return path_name
+
+
+def loop_directories_children(
+    dir: Path, env_expected_dir_variable: Dict[str, Path]
+) -> Dict[str, Path]:
+    """Loop through a directories children and add their paths to a dictionary.
+
+    Args:
+        dir: directory containing Children locations
+        env_expected_dir_variable : Dictionary where the keys are directory
+            variables and values are their paths.
+
+    Returns:
+        Dictionary of directory keys and their paths with the children of the
+        given directory added.
+    """
+    for child in dir.iterdir():
+        if child.is_dir():
+            dir_name = remove_brackets_and_spaces(dir.name.upper())
+            child_name = remove_brackets_and_spaces(child.name.upper())
+            env_expected_dir_variable[f"DIR_{dir_name}_{child_name}"] = child
+    return env_expected_dir_variable
+
+
 def get_actual_env_variables(path_env: Path) -> Dict[str, Path]:
     """Get the environment variables and values for directories in the `.env` file of
      the `govcookiecutter` template.
@@ -61,7 +101,7 @@ def define_expected_env_variables(
     # upper case in the format "DIR_<<<DIRECTORY_NAME>>>". Ignore any directories
     # with a name in `exclude_root_folders`
     env_expected_dir_variable = {
-        f"DIR_{d.name.upper()}": d
+        f"DIR_{remove_brackets_and_spaces(d.name.upper())}": d
         for d in folder.glob("*")
         if d.is_dir() and d.name not in exclude_root_folders
     }
@@ -77,9 +117,14 @@ def define_expected_env_variables(
             and d.name not in exclude_folders
             and d.parent.name not in exclude_sub_folders_in_parent_folders
         ):
-            env_expected_dir_variable[
-                f"DIR_{d.parent.name.upper()}_{d.name.upper()}"
-            ] = d
+            if d.name.upper() == "SRC":
+                env_expected_dir_variable = loop_directories_children(
+                    d, env_expected_dir_variable
+                )
+
+            parent = remove_brackets_and_spaces(d.parent.name.upper())
+            name = remove_brackets_and_spaces(d.name.upper())
+            env_expected_dir_variable[f"DIR_{parent}_{name}"] = d
     return env_expected_dir_variable
 
 

tests/test_govcookiecutter_creation.py

@@ -1,7 +1,8 @@
-from sphinx.cmd.build import main
+import re
 from typing import Dict
+
 import pytest
-import re
+from sphinx.cmd.build import main
 
 
 @pytest.mark.parametrize("test_input_repository_hosting_platform", ["GitHub", "GitLab"])