From ef7990d7d00afbb5dbeef3bdbbc348f7f60c8f3c Mon Sep 17 00:00:00 2001 From: joshlynchONS <118733244+joshlynchONS@users.noreply.github.com> Date: Wed, 15 Feb 2023 15:21:02 +0000 Subject: [PATCH 01/13] add: update folder structure and setup files so python package is automatically installable * add auto generating changlog to workflow * add auto generating changlog to workflow * changed to checkoutv3 in changlog workflow * adding initial package files for user * changed file structure for package and changed package name to repo name * change expected values for repo_name * change expected values for repo_name * change expected values for repo_name * change expected values for repo_name * changed file structure for package and changed package name to repo name * changed file structure for package and changed package name to repo name * changed file structure and changed directory tests in env * changed directory tests in env * changed directory tests in env * changing ignored subfolders in env test * changing ignored subfolders and env variable names * changing ignored subfolders and env variable names * changing name change in env test * added expected paths for folders in src * added expected paths for folders in src * changed src and tests env directories * changed src and tests env directories and hid repo name in expected paths * added bug fix to make expected repo name upper * adding environment variable for package dir * adding environment variable for package dir * removing brackets from directory variables to match env file * removing brackets and spaces from directory variables to match env file * removing brackets and spaces from directory variables to match env file * fix bug in remove bracket and space function * change format of env variables * refactoring test_env * updating initial requiremnets files and package location * updating typo in testing injected variables * updated main readme to include package setup details * changed: updated setup and requirement files * docs: update structure in readmes * docs: remove src package reference * comment out src package in docs * comment out src package in docs * chore: remove fast fail from workflow * chore: remove fast fail * Revert "comment out src package in docs" revert document changes This reverts commit 23f23ab08f430019d85e6abb8f40066b9d182429. * revert "comment out src package in docs" This reverts commit 5a804824799f5f4b035358fc7f56df72b8a97613. * Revert "docs: remove src package reference" This reverts commit 24c135848c15a450fdc70c0cab9cc23fbaf341df. * Revert "docs: update structure in readmes" This reverts commit d523e81565b045e6e3e4996424b37addf8af6307. * docs: change src module to {{ cookiecutter.repo_name }} * fix: update injected variable count * fix: update injected variable count * Revert "changed to checkoutv3 in changlog workflow" This reverts commit 10ac908549226cacaab9f76638bf31f61f4be7aa. * fix: remove changelog workflow * docs: move package install instructions to template readme * fix: remove changelog workflow * docs: removed src and test docs in structure documentation and replaced with package name * fix: update version of isort to 5.12.0 for poetry fix * fix: update python versions in build wf's as isort no longer compatible with 3.7 * fix: fix python version 3.10 in build with ' * fix: fix accessability link * fix: update isort on template build * fix: remove r version 4.0.4 * fix: remove file exclusion from pre commit * fix: remove ubuntu from template build * fix: remove R version 4.05 * fix: update R versions in template build * fix: add ubuntu os to template build * chore: remove lower version of R for test * fix: update dependencies for ubuntu in template build * fix: add dependency to ubuntu packages * fix: remove dependency to test if test passes * fix: update os in template build * docs: update docs to mention python installable and R not --- .github/workflows/govcookiecutter-build.yml | 2 +- .../govcookiecutter-template-build.yml | 9 ++-- .pre-commit-config.yaml | 2 +- README.md | 3 +- docs/accessibility.md | 2 +- tests/test_env.py | 52 +++++++++++++++++-- ...test_govcookiecutter_injected_variables.py | 11 ++-- {{ cookiecutter.repo_name }}/.env | 17 +++--- .../.pre-commit-config.yaml | 2 +- {{ cookiecutter.repo_name }}/README.md | 15 ++++++ .../writing_sphinx_documentation.md | 12 ++--- .../docs/reference/make_data.md | 4 +- .../docs/reference/make_features.md | 4 +- .../docs/reference/make_models.md | 4 +- .../docs/reference/make_visualisations.md | 4 +- .../docs/reference/src.md | 4 +- .../docs/reference/utils.md | 4 +- .../docs/structure/src.md | 2 - .../docs/structure/tests.md | 2 - .../structure/{{ cookiecutter.repo_name }}.md | 3 ++ {{ cookiecutter.repo_name }}/pyproject.toml | 4 +- {{ cookiecutter.repo_name }}/setup.cfg | 27 ++++++++++ {{ cookiecutter.repo_name }}/setup.py | 4 ++ .../__init__.py | 0 .../src/README.md | 0 .../src}/__init__.py | 0 .../src/make_data}/__init__.py | 0 .../src/make_features}/__init__.py | 0 .../src/make_models}/__init__.py | 0 .../src/make_visualisations}/__init__.py | 0 .../src/utils/__init__.py | 0 .../tests/README.md | 0 .../tests/__init__.py | 0 33 files changed, 145 insertions(+), 48 deletions(-) delete mode 100644 {{ cookiecutter.repo_name }}/docs/structure/src.md delete mode 100644 {{ cookiecutter.repo_name }}/docs/structure/tests.md create mode 100644 {{ cookiecutter.repo_name }}/docs/structure/{{ cookiecutter.repo_name }}.md create mode 100644 {{ cookiecutter.repo_name }}/setup.cfg create mode 100644 {{ cookiecutter.repo_name }}/setup.py rename {{ cookiecutter.repo_name }}/{src => {{ cookiecutter.repo_name }}}/__init__.py (100%) rename {{ cookiecutter.repo_name }}/{ => {{ cookiecutter.repo_name }}}/src/README.md (100%) rename {{ cookiecutter.repo_name }}/{src/make_data => {{ cookiecutter.repo_name }}/src}/__init__.py (100%) rename {{ cookiecutter.repo_name }}/{src/make_features => {{ cookiecutter.repo_name }}/src/make_data}/__init__.py (100%) rename {{ cookiecutter.repo_name }}/{src/make_models => {{ cookiecutter.repo_name }}/src/make_features}/__init__.py (100%) rename {{ cookiecutter.repo_name }}/{src/make_visualisations => {{ cookiecutter.repo_name }}/src/make_models}/__init__.py (100%) rename {{ cookiecutter.repo_name }}/{src/utils => {{ cookiecutter.repo_name }}/src/make_visualisations}/__init__.py (100%) create mode 100644 {{ cookiecutter.repo_name }}/{{ cookiecutter.repo_name }}/src/utils/__init__.py rename {{ cookiecutter.repo_name }}/{ => {{ cookiecutter.repo_name }}}/tests/README.md (100%) create mode 100644 {{ cookiecutter.repo_name }}/{{ cookiecutter.repo_name }}/tests/__init__.py diff --git a/.github/workflows/govcookiecutter-build.yml b/.github/workflows/govcookiecutter-build.yml index ec355f1..4a3fdf5 100644 --- a/.github/workflows/govcookiecutter-build.yml +++ b/.github/workflows/govcookiecutter-build.yml @@ -9,7 +9,7 @@ jobs: strategy: matrix: os: [ ubuntu-latest, macos-latest, windows-latest ] - python: [ 3.7, 3.8, 3.9, '3.10' ] + python: [ 3.8, 3.9, '3.10', 3.11 ] steps: - name: Checkout the revision diff --git a/.github/workflows/govcookiecutter-template-build.yml b/.github/workflows/govcookiecutter-template-build.yml index 9d54297..1668c38 100644 --- a/.github/workflows/govcookiecutter-template-build.yml +++ b/.github/workflows/govcookiecutter-template-build.yml @@ -10,12 +10,11 @@ jobs: matrix: # TODO: Fix R pre-commit hook issues with Windows os: [ ubuntu-latest, macos-latest ] - python: [ 3.7, 3.8, 3.9, '3.10' ] + python: [ 3.8, 3.9, '3.10' ] R: - {using_R: No, version: N/A} - - {using_R: Yes, version: 4.0.4} - - {using_R: Yes, version: 4.0.5} - - {using_R: Yes, version: 4.1.0} + - {using_R: Yes, version: 4.1.3} + - {using_R: Yes, version: 4.2.2} steps: - name: Checkout the revision @@ -33,7 +32,9 @@ jobs: if: ${{ matrix.R.using_r == 'Yes' }} - name: Install other ${{ matrix.os }} R dependencies run: | + sudo apt-get update sudo apt-get install libcurl4-openssl-dev + sudo apt-get install libgit2-dev sudo apt-get install libharfbuzz-dev sudo apt-get install libfribidi-dev if: ${{ matrix.os == 'ubuntu-latest' && matrix.R.using_R == 'Yes' }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 507a394..d292129 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -12,7 +12,7 @@ repos: - id: trailing-whitespace name: Check for trailing whitespaces (auto-fixes) - repo: https://github.com/pycqa/isort - rev: 5.8.0 + rev: 5.12.0 hooks: - id: isort name: isort - Sort Python imports (auto-fixes) diff --git a/README.md b/README.md index 455fa99..15d7421 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,8 @@ A cookiecutter template for analytical, Python-, or Python and R-based projects His Majesty's Government, and wider public sector. This template helps to set up standardised project structures, and [includes security -features using pre-commit hooks][docs-pre-commit]. +features using pre-commit hooks][docs-pre-commit]. This cookiecutter template also acts +as an installable template (python projects only). It also provides an Agile, centralised, and lightweight analytical quality assurance (AQA) process. Pull or merge request templates are used to nudge users to complete this diff --git a/docs/accessibility.md b/docs/accessibility.md index 9f22d69..b3aea90 100644 --- a/docs/accessibility.md +++ b/docs/accessibility.md @@ -104,7 +104,7 @@ We plan to fix the accessibility issues in the content by the end of December 20 This statement was prepared on 30 June 2021. It was last reviewed on 20 July 2021. -[abilitynet]: https://mcmw.abilitynet.org.uk/ +[abilitynet]: https://abilitynet.org.uk/ [accessibility-legislation]: https://www.legislation.gov.uk/uksi/2018/952/regulation/4/made [eass]: https://www.equalityadvisoryservice.com/ [email]: mailto:gsshelp@statistics.gov.uk diff --git a/tests/test_env.py b/tests/test_env.py index f3665db..cb0310b 100644 --- a/tests/test_env.py +++ b/tests/test_env.py @@ -14,6 +14,45 @@ EXCLUDE_SUB_DIR_IN_PARENTS_NAMES = [*EXCLUDE_ROOT_DIR_NAMES, "docs"] +def remove_brackets_and_spaces(path_name: str) -> str: + """Removes spaces and brackets from a string. + + Args: + path_name : A path name as a string. + + Returns: + The path name with spaces and brackets removed. + """ + if "{" in path_name or "}" in path_name or " " in path_name: + path_name = path_name.replace("{", "") + path_name = path_name.replace("}", "") + path_name = path_name.replace(" ", "") + + return path_name + + +def loop_directories_children( + dir: Path, env_expected_dir_variable: Dict[str, Path] +) -> Dict[str, Path]: + """Loop through a directories children and add their paths to a dictionary. + + Args: + dir: directory containing Children locations + env_expected_dir_variable : Dictionary where the keys are directory + variables and values are their paths. + + Returns: + Dictionary of directory keys and their paths with the children of the + given directory added. + """ + for child in dir.iterdir(): + if child.is_dir(): + dir_name = remove_brackets_and_spaces(dir.name.upper()) + child_name = remove_brackets_and_spaces(child.name.upper()) + env_expected_dir_variable[f"DIR_{dir_name}_{child_name}"] = child + return env_expected_dir_variable + + def get_actual_env_variables(path_env: Path) -> Dict[str, Path]: """Get the environment variables and values for directories in the `.env` file of the `govcookiecutter` template. @@ -61,7 +100,7 @@ def define_expected_env_variables( # upper case in the format "DIR_<<>>". Ignore any directories # with a name in `exclude_root_folders` env_expected_dir_variable = { - f"DIR_{d.name.upper()}": d + f"DIR_{remove_brackets_and_spaces(d.name.upper())}": d for d in folder.glob("*") if d.is_dir() and d.name not in exclude_root_folders } @@ -77,9 +116,14 @@ def define_expected_env_variables( and d.name not in exclude_folders and d.parent.name not in exclude_sub_folders_in_parent_folders ): - env_expected_dir_variable[ - f"DIR_{d.parent.name.upper()}_{d.name.upper()}" - ] = d + if d.name.upper() == "SRC": + env_expected_dir_variable = loop_directories_children( + d, env_expected_dir_variable + ) + + parent = remove_brackets_and_spaces(d.parent.name.upper()) + name = remove_brackets_and_spaces(d.name.upper()) + env_expected_dir_variable[f"DIR_{parent}_{name}"] = d return env_expected_dir_variable diff --git a/tests/test_govcookiecutter_injected_variables.py b/tests/test_govcookiecutter_injected_variables.py index 54d6cec..dcdd91b 100644 --- a/tests/test_govcookiecutter_injected_variables.py +++ b/tests/test_govcookiecutter_injected_variables.py @@ -30,13 +30,16 @@ } REPO_NAME_COUNT = { '"{{ cookiecutter.repo_name }}",': 0, - "`{{ cookiecutter.repo_name }}`": 7, + "`{{ cookiecutter.repo_name }}`": 15, "`{{ cookiecutter.repo_name }}`,": 1, '"{{ cookiecutter.repo_name }}.tex",': 0, '"{{ cookiecutter.repo_name }}doc"': 1, - "{{ cookiecutter.repo_name }}": 0, + "{{ cookiecutter.repo_name }}": 10, +} +OVERVIEW_COUNT = { + '"{{ cookiecutter.overview }}",': 0, + "{{ cookiecutter.overview }}": 1, } -OVERVIEW_COUNT = {'"{{ cookiecutter.overview }}",': 0, "{{ cookiecutter.overview }}": 1} PROJECT_VERSION_COUNT = { '"{{ cookiecutter.project_version }}"': 2, "{{ cookiecutter.project_version }}": 0, @@ -162,7 +165,7 @@ def recursive_open_and_count_search_terms( ( "repo_name", "repo_2", - {**REPO_NAME_COUNT, "{{ cookiecutter.repo_name }}": 1}, + {**REPO_NAME_COUNT, "{{ cookiecutter.repo_name }}": 11}, {"using_R": "Yes"}, ), ("overview", "overview_1", OVERVIEW_COUNT, {"using_R": "No"}), diff --git a/{{ cookiecutter.repo_name }}/.env b/{{ cookiecutter.repo_name }}/.env index f3fe491..739bda8 100644 --- a/{{ cookiecutter.repo_name }}/.env +++ b/{{ cookiecutter.repo_name }}/.env @@ -38,13 +38,16 @@ DIR_NOTEBOOKS=./notebooks # Add environment variables for the `outputs` directory DIR_OUTPUTS=./outputs +# Add environment variable for package directory +DIR_COOKIECUTTER.REPO_NAME = ./{{ cookiecutter.repo_name }} + # Add environment variables for the `src` directories -DIR_SRC=./src -DIR_SRC_MAKE_DATA=./src/make_data -DIR_SRC_MAKE_FEATURES=./src/make_features -DIR_SRC_MAKE_MODELS=./src/make_models -DIR_SRC_MAKE_VISUALISATIONS=./src/make_visualisations -DIR_SRC_UTILS=./src/utils +DIR_COOKIECUTTER.REPO_NAME_SRC=./{{ cookiecutter.repo_name }}/src +DIR_SRC_MAKE_DATA=./{{ cookiecutter.repo_name }}/src/make_data +DIR_SRC_MAKE_FEATURES=./{{ cookiecutter.repo_name }}/src/make_features +DIR_SRC_MAKE_MODELS=./{{ cookiecutter.repo_name }}/src/make_models +DIR_SRC_MAKE_VISUALISATIONS=./{{ cookiecutter.repo_name }}/src/make_visualisations +DIR_SRC_UTILS=./{{ cookiecutter.repo_name }}/src/utils # Add environment variables for the `tests` directory -DIR_TESTS=./tests +DIR_COOKIECUTTER.REPO_NAME_TESTS=./{{ cookiecutter.repo_name }}/tests diff --git a/{{ cookiecutter.repo_name }}/.pre-commit-config.yaml b/{{ cookiecutter.repo_name }}/.pre-commit-config.yaml index 95e5f78..63fc530 100644 --- a/{{ cookiecutter.repo_name }}/.pre-commit-config.yaml +++ b/{{ cookiecutter.repo_name }}/.pre-commit-config.yaml @@ -21,7 +21,7 @@ repos: - id: trailing-whitespace name: Check for trailing whitespaces (auto-fixes) - repo: https://github.com/pycqa/isort - rev: 5.8.0 + rev: 5.12.0 hooks: - id: isort name: isort - Sort Python imports (auto-fixes) diff --git a/{{ cookiecutter.repo_name }}/README.md b/{{ cookiecutter.repo_name }}/README.md index ff4cc3b..9836f6a 100644 --- a/{{ cookiecutter.repo_name }}/README.md +++ b/{{ cookiecutter.repo_name }}/README.md @@ -32,6 +32,21 @@ To install the Python requirements, open your terminal and enter: pip install -r requirements.txt ``` +## Installing your package (Python Only) + +Whilst in the root folder, in the command prompt, you can install your package using: + +```shell +pip install -e . +``` + +This installs an editable version of your package. Meaning, when you update your +package code, you do not have to reinstall it for the changes to take effect. +(This saves a lot of time when you test your code) + +Remember to update the setup and requirement files inline with any changes to your +package. The inital files contain the bare minimum to get you started. + ## Required secrets and credentials To run this project, [you need a `.secrets` file with secrets/credentials as diff --git a/{{ cookiecutter.repo_name }}/docs/contributor_guide/writing_sphinx_documentation.md b/{{ cookiecutter.repo_name }}/docs/contributor_guide/writing_sphinx_documentation.md index cc80669..89d232c 100644 --- a/{{ cookiecutter.repo_name }}/docs/contributor_guide/writing_sphinx_documentation.md +++ b/{{ cookiecutter.repo_name }}/docs/contributor_guide/writing_sphinx_documentation.md @@ -47,12 +47,12 @@ the next subsection. ### Automatically creating docstring documentation (ReST) -Let us say that `src/__init__.py` has functions called `hello` and `world` imported +Let us say that `{{ cookiecutter.repo_name }}/__init__.py` has functions called `hello` and `world` imported into it, and both have docstrings. To automatically generate docstring documentation, -create a ReST file, and add the following line to reference the `src` module: +create a ReST file, and add the following line to reference the `{{ cookiecutter.repo_name }}` module: ```rest -.. currentmodule:: src +.. currentmodule:: {{ cookiecutter.repo_name }} ``` Then, elsewhere in the body, [call the `autosummary` directive to generate the @@ -82,13 +82,13 @@ it. We will cover some of the more widely used elements in the following subsect ### Automatically creating docstring documentation (MyST Markdown) -Let us say that `src/__init__.py` has functions called `hello` and `world` imported +Let us say that `{{ cookiecutter.repo_name }}/__init__.py` has functions called `hello` and `world` imported into it, and both have docstrings. To automatically generate docstring documentation, -create a Markdown file, and add the following line to reference the `src` module: +create a Markdown file, and add the following line to reference the `{{ cookiecutter.repo_name }}` module: ````md ```{eval-rst} -.. currentmodule:: src +.. currentmodule:: {{ cookiecutter.repo_name }} ``` ```` diff --git a/{{ cookiecutter.repo_name }}/docs/reference/make_data.md b/{{ cookiecutter.repo_name }}/docs/reference/make_data.md index 3411b49..4adfdbe 100644 --- a/{{ cookiecutter.repo_name }}/docs/reference/make_data.md +++ b/{{ cookiecutter.repo_name }}/docs/reference/make_data.md @@ -1,9 +1,9 @@ # Data generation -These `src` package functions generate data. +These `{{ cookiecutter.repo_name }}` package functions generate data. ```{eval-rst} -.. currentmodule:: src +.. currentmodule:: {{ cookiecutter.repo_name }} ``` ## Function heading diff --git a/{{ cookiecutter.repo_name }}/docs/reference/make_features.md b/{{ cookiecutter.repo_name }}/docs/reference/make_features.md index 359be20..7beaef6 100644 --- a/{{ cookiecutter.repo_name }}/docs/reference/make_features.md +++ b/{{ cookiecutter.repo_name }}/docs/reference/make_features.md @@ -1,9 +1,9 @@ # Feature generation -These `src` package functions create features. +These `{{ cookiecutter.repo_name }}` package functions create features. ```{eval-rst} -.. currentmodule:: src +.. currentmodule:: {{ cookiecutter.repo_name }} ``` ## Function heading diff --git a/{{ cookiecutter.repo_name }}/docs/reference/make_models.md b/{{ cookiecutter.repo_name }}/docs/reference/make_models.md index e1fa8e8..1a607a4 100644 --- a/{{ cookiecutter.repo_name }}/docs/reference/make_models.md +++ b/{{ cookiecutter.repo_name }}/docs/reference/make_models.md @@ -1,9 +1,9 @@ # Model generation -These `src` package functions create models. +These `{{ cookiecutter.repo_name }}` package functions create models. ```{eval-rst} -.. currentmodule:: src +.. currentmodule:: {{ cookiecutter.repo_name }} ``` ## Function heading diff --git a/{{ cookiecutter.repo_name }}/docs/reference/make_visualisations.md b/{{ cookiecutter.repo_name }}/docs/reference/make_visualisations.md index 323a950..72056f8 100644 --- a/{{ cookiecutter.repo_name }}/docs/reference/make_visualisations.md +++ b/{{ cookiecutter.repo_name }}/docs/reference/make_visualisations.md @@ -1,9 +1,9 @@ # Create visualisations -These `src` package functions create visualisations. +These `{{ cookiecutter.repo_name }}` package functions create visualisations. ```{eval-rst} -.. currentmodule:: src +.. currentmodule:: {{ cookiecutter.repo_name }} ``` ## Function heading diff --git a/{{ cookiecutter.repo_name }}/docs/reference/src.md b/{{ cookiecutter.repo_name }}/docs/reference/src.md index cfbcb90..a4b4c4f 100644 --- a/{{ cookiecutter.repo_name }}/docs/reference/src.md +++ b/{{ cookiecutter.repo_name }}/docs/reference/src.md @@ -1,9 +1,9 @@ # Package functions -These are miscellaneous functions in the `src` package. +These are miscellaneous functions in the `{{ cookiecutter.repo_name }}` package. ```{eval-rst} -.. currentmodule:: src +.. currentmodule:: {{ cookiecutter.repo_name }} ``` ## Function heading diff --git a/{{ cookiecutter.repo_name }}/docs/reference/utils.md b/{{ cookiecutter.repo_name }}/docs/reference/utils.md index 640a975..2dbfe22 100644 --- a/{{ cookiecutter.repo_name }}/docs/reference/utils.md +++ b/{{ cookiecutter.repo_name }}/docs/reference/utils.md @@ -1,9 +1,9 @@ # Create utility functions -These `src` package functions create utility functions. +These `{{ cookiecutter.repo_name }}` package functions create utility functions. ```{eval-rst} -.. currentmodule:: src +.. currentmodule:: {{ cookiecutter.repo_name }} ``` ## Function heading diff --git a/{{ cookiecutter.repo_name }}/docs/structure/src.md b/{{ cookiecutter.repo_name }}/docs/structure/src.md deleted file mode 100644 index 6c3a21c..0000000 --- a/{{ cookiecutter.repo_name }}/docs/structure/src.md +++ /dev/null @@ -1,2 +0,0 @@ -```{include} ../../src/README.md -``` diff --git a/{{ cookiecutter.repo_name }}/docs/structure/tests.md b/{{ cookiecutter.repo_name }}/docs/structure/tests.md deleted file mode 100644 index 66d3ee5..0000000 --- a/{{ cookiecutter.repo_name }}/docs/structure/tests.md +++ /dev/null @@ -1,2 +0,0 @@ -```{include} ../../tests/README.md -``` diff --git a/{{ cookiecutter.repo_name }}/docs/structure/{{ cookiecutter.repo_name }}.md b/{{ cookiecutter.repo_name }}/docs/structure/{{ cookiecutter.repo_name }}.md new file mode 100644 index 0000000..6472d67 --- /dev/null +++ b/{{ cookiecutter.repo_name }}/docs/structure/{{ cookiecutter.repo_name }}.md @@ -0,0 +1,3 @@ +```{include} ../../{{ cookiecutter.repo_name }}/README.md +:relative-docs: ../docs/structure +``` diff --git a/{{ cookiecutter.repo_name }}/pyproject.toml b/{{ cookiecutter.repo_name }}/pyproject.toml index d8d811a..fb0484a 100644 --- a/{{ cookiecutter.repo_name }}/pyproject.toml +++ b/{{ cookiecutter.repo_name }}/pyproject.toml @@ -1,7 +1,7 @@ # `coverage` configurations [tool.coverage.run] source = [ - "./src" + "./" ] [tool.coverage.report] @@ -21,5 +21,5 @@ addopts = [ ] doctest_optionflags = "NORMALIZE_WHITESPACE" testpaths = [ - "./tests" + "./{{ cookiecutter.repo_name }}/tests" ] diff --git a/{{ cookiecutter.repo_name }}/setup.cfg b/{{ cookiecutter.repo_name }}/setup.cfg new file mode 100644 index 0000000..dd957f6 --- /dev/null +++ b/{{ cookiecutter.repo_name }}/setup.cfg @@ -0,0 +1,27 @@ +[metadata] +name = {{ cookiecutter.repo_name }} +description = a brief description +author = {{ cookiecutter.organisation_handle }} +platforms = win32 +classifiers = + Programming Language :: Python :: 3.6 + Programming Language :: Python :: 3.7 + Programming Language :: Python :: 3.8 + Programming Language :: Python :: 3.9 + +[options] +packages = + {{ cookiecutter.repo_name }} +install_requires = + coverage + detect-secrets == 1.0.3 + myst-parser + pre-commit + pytest + detect-secrets + python-dotenv + Sphinx + toml +python_requires = >=3.6 + +zip_safe = no diff --git a/{{ cookiecutter.repo_name }}/setup.py b/{{ cookiecutter.repo_name }}/setup.py new file mode 100644 index 0000000..7f1a176 --- /dev/null +++ b/{{ cookiecutter.repo_name }}/setup.py @@ -0,0 +1,4 @@ +from setuptools import setup + +if __name__ == "__main__": + setup() diff --git a/{{ cookiecutter.repo_name }}/src/__init__.py b/{{ cookiecutter.repo_name }}/{{ cookiecutter.repo_name }}/__init__.py similarity index 100% rename from {{ cookiecutter.repo_name }}/src/__init__.py rename to {{ cookiecutter.repo_name }}/{{ cookiecutter.repo_name }}/__init__.py diff --git a/{{ cookiecutter.repo_name }}/src/README.md b/{{ cookiecutter.repo_name }}/{{ cookiecutter.repo_name }}/src/README.md similarity index 100% rename from {{ cookiecutter.repo_name }}/src/README.md rename to {{ cookiecutter.repo_name }}/{{ cookiecutter.repo_name }}/src/README.md diff --git a/{{ cookiecutter.repo_name }}/src/make_data/__init__.py b/{{ cookiecutter.repo_name }}/{{ cookiecutter.repo_name }}/src/__init__.py similarity index 100% rename from {{ cookiecutter.repo_name }}/src/make_data/__init__.py rename to {{ cookiecutter.repo_name }}/{{ cookiecutter.repo_name }}/src/__init__.py diff --git a/{{ cookiecutter.repo_name }}/src/make_features/__init__.py b/{{ cookiecutter.repo_name }}/{{ cookiecutter.repo_name }}/src/make_data/__init__.py similarity index 100% rename from {{ cookiecutter.repo_name }}/src/make_features/__init__.py rename to {{ cookiecutter.repo_name }}/{{ cookiecutter.repo_name }}/src/make_data/__init__.py diff --git a/{{ cookiecutter.repo_name }}/src/make_models/__init__.py b/{{ cookiecutter.repo_name }}/{{ cookiecutter.repo_name }}/src/make_features/__init__.py similarity index 100% rename from {{ cookiecutter.repo_name }}/src/make_models/__init__.py rename to {{ cookiecutter.repo_name }}/{{ cookiecutter.repo_name }}/src/make_features/__init__.py diff --git a/{{ cookiecutter.repo_name }}/src/make_visualisations/__init__.py b/{{ cookiecutter.repo_name }}/{{ cookiecutter.repo_name }}/src/make_models/__init__.py similarity index 100% rename from {{ cookiecutter.repo_name }}/src/make_visualisations/__init__.py rename to {{ cookiecutter.repo_name }}/{{ cookiecutter.repo_name }}/src/make_models/__init__.py diff --git a/{{ cookiecutter.repo_name }}/src/utils/__init__.py b/{{ cookiecutter.repo_name }}/{{ cookiecutter.repo_name }}/src/make_visualisations/__init__.py similarity index 100% rename from {{ cookiecutter.repo_name }}/src/utils/__init__.py rename to {{ cookiecutter.repo_name }}/{{ cookiecutter.repo_name }}/src/make_visualisations/__init__.py diff --git a/{{ cookiecutter.repo_name }}/{{ cookiecutter.repo_name }}/src/utils/__init__.py b/{{ cookiecutter.repo_name }}/{{ cookiecutter.repo_name }}/src/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/{{ cookiecutter.repo_name }}/tests/README.md b/{{ cookiecutter.repo_name }}/{{ cookiecutter.repo_name }}/tests/README.md similarity index 100% rename from {{ cookiecutter.repo_name }}/tests/README.md rename to {{ cookiecutter.repo_name }}/{{ cookiecutter.repo_name }}/tests/README.md diff --git a/{{ cookiecutter.repo_name }}/{{ cookiecutter.repo_name }}/tests/__init__.py b/{{ cookiecutter.repo_name }}/{{ cookiecutter.repo_name }}/tests/__init__.py new file mode 100644 index 0000000..e69de29 From 79dfe60b9f92a2f359571e820eeebf683e29edfc Mon Sep 17 00:00:00 2001 From: joshlynchONS <118733244+joshlynchONS@users.noreply.github.com> Date: Mon, 6 Mar 2023 12:03:49 +0000 Subject: [PATCH 02/13] Add: Updating folder structure * fix: remove data sub folders and update env variables * fix: remove unnecessary root folders and update environment variables * fix: rearrange package and src and update env variables * fix: fix test env variable * fix: update setup files for install * fix: add example test to env variables * Revert "fix: add example test to env variables" This reverts commit eb306638fde955664ee2ad2b68ecd7956516cc66. * Revert "fix: update setup files for install" This reverts commit 1021857ab36e1e1727e683af229b3204bf0b256d. * Revert "fix: fix test env variable" This reverts commit 27745eeda354861ed15bbdcc79834562e1177be0. * Revert "fix: rearrange package and src and update env variables" This reverts commit 7124468ee404c2dd992ce0ff3b072db82d4cb78b. * fix: move test folder and change test env variable * fix: remove module folders from package and update env variables * fix: swap src and package folders around and update env variables * fix: remove utils folder env variable * docs: remove structure documentation * fix: update injected variables for structure docs removal * docs: remove reference docs for package * fix: update injected variables for reference folder removal * fix: update repo name count in injected variables * docs: update docs to reflect the removed folders * fix: remove autosummary code blocks in writing sphinx docs * fix: updated package location for pip install * fix: update repo injected variables * fix: fix directory for setup.cfg * fix: update injected variable counts --- ...test_govcookiecutter_injected_variables.py | 16 +- {{ cookiecutter.repo_name }}/.env | 21 +- {{ cookiecutter.repo_name }}/data/README.md | 19 +- .../data/external/.gitkeep | 0 .../data/interim/.gitkeep | 0 .../data/processed/.gitkeep | 0 .../data/raw/.gitkeep | 0 .../writing_sphinx_documentation.md | 20 -- {{ cookiecutter.repo_name }}/docs/index.md | 2 - .../docs/reference/README.md | 14 -- .../docs/reference/make_data.md | 15 -- .../docs/reference/make_features.md | 15 -- .../docs/reference/make_models.md | 15 -- .../docs/reference/make_visualisations.md | 15 -- .../docs/reference/src.md | 15 -- .../docs/reference/utils.md | 15 -- .../docs/structure/README.md | 203 ------------------ .../docs/structure/data.md | 3 - .../docs/structure/docs.md | 3 - .../docs/structure/notebooks.md | 3 - .../docs/structure/outputs.md | 3 - .../structure/{{ cookiecutter.repo_name }}.md | 3 - .../notebooks/README.md | 7 - .../outputs/README.md | 12 -- {{ cookiecutter.repo_name }}/pyproject.toml | 2 +- {{ cookiecutter.repo_name }}/setup.cfg | 3 +- .../__init__.py | 0 .../{{ cookiecutter.repo_name }}}/__init__.py | 0 .../utils}/__init__.py | 0 .../tests/README.md | 0 .../src/make_features => tests}/__init__.py | 0 .../src/README.md | 24 --- .../src/make_models/__init__.py | 0 .../src/make_visualisations/__init__.py | 0 .../src/utils/__init__.py | 0 .../tests/__init__.py | 0 36 files changed, 19 insertions(+), 429 deletions(-) delete mode 100644 {{ cookiecutter.repo_name }}/data/external/.gitkeep delete mode 100644 {{ cookiecutter.repo_name }}/data/interim/.gitkeep delete mode 100644 {{ cookiecutter.repo_name }}/data/processed/.gitkeep delete mode 100644 {{ cookiecutter.repo_name }}/data/raw/.gitkeep delete mode 100644 {{ cookiecutter.repo_name }}/docs/reference/README.md delete mode 100644 {{ cookiecutter.repo_name }}/docs/reference/make_data.md delete mode 100644 {{ cookiecutter.repo_name }}/docs/reference/make_features.md delete mode 100644 {{ cookiecutter.repo_name }}/docs/reference/make_models.md delete mode 100644 {{ cookiecutter.repo_name }}/docs/reference/make_visualisations.md delete mode 100644 {{ cookiecutter.repo_name }}/docs/reference/src.md delete mode 100644 {{ cookiecutter.repo_name }}/docs/reference/utils.md delete mode 100644 {{ cookiecutter.repo_name }}/docs/structure/README.md delete mode 100644 {{ cookiecutter.repo_name }}/docs/structure/data.md delete mode 100644 {{ cookiecutter.repo_name }}/docs/structure/docs.md delete mode 100644 {{ cookiecutter.repo_name }}/docs/structure/notebooks.md delete mode 100644 {{ cookiecutter.repo_name }}/docs/structure/outputs.md delete mode 100644 {{ cookiecutter.repo_name }}/docs/structure/{{ cookiecutter.repo_name }}.md delete mode 100644 {{ cookiecutter.repo_name }}/notebooks/README.md delete mode 100644 {{ cookiecutter.repo_name }}/outputs/README.md rename {{ cookiecutter.repo_name }}/{{{ cookiecutter.repo_name }} => src}/__init__.py (100%) rename {{ cookiecutter.repo_name }}/{{{ cookiecutter.repo_name }}/src => src/{{ cookiecutter.repo_name }}}/__init__.py (100%) rename {{ cookiecutter.repo_name }}/{{{ cookiecutter.repo_name }}/src/make_data => src/{{ cookiecutter.repo_name }}/utils}/__init__.py (100%) rename {{ cookiecutter.repo_name }}/{{{ cookiecutter.repo_name }} => }/tests/README.md (100%) rename {{ cookiecutter.repo_name }}/{{{ cookiecutter.repo_name }}/src/make_features => tests}/__init__.py (100%) delete mode 100644 {{ cookiecutter.repo_name }}/{{ cookiecutter.repo_name }}/src/README.md delete mode 100644 {{ cookiecutter.repo_name }}/{{ cookiecutter.repo_name }}/src/make_models/__init__.py delete mode 100644 {{ cookiecutter.repo_name }}/{{ cookiecutter.repo_name }}/src/make_visualisations/__init__.py delete mode 100644 {{ cookiecutter.repo_name }}/{{ cookiecutter.repo_name }}/src/utils/__init__.py delete mode 100644 {{ cookiecutter.repo_name }}/{{ cookiecutter.repo_name }}/tests/__init__.py diff --git a/tests/test_govcookiecutter_injected_variables.py b/tests/test_govcookiecutter_injected_variables.py index dcdd91b..578db64 100644 --- a/tests/test_govcookiecutter_injected_variables.py +++ b/tests/test_govcookiecutter_injected_variables.py @@ -12,7 +12,7 @@ } ORGANISATION_HANDLE_COUNT = { '"{{ cookiecutter.organisation_handle }}"': 1, - "`{{ cookiecutter.organisation_handle }}`": 4, + "`{{ cookiecutter.organisation_handle }}`": 3, 'u"{{ cookiecutter.organisation_handle }}",': 0, '[u"{{ cookiecutter.organisation_handle }}"],': 0, '"{{ cookiecutter.organisation_handle }}",': 0, @@ -30,11 +30,11 @@ } REPO_NAME_COUNT = { '"{{ cookiecutter.repo_name }}",': 0, - "`{{ cookiecutter.repo_name }}`": 15, + "`{{ cookiecutter.repo_name }}`": 8, "`{{ cookiecutter.repo_name }}`,": 1, '"{{ cookiecutter.repo_name }}.tex",': 0, '"{{ cookiecutter.repo_name }}doc"': 1, - "{{ cookiecutter.repo_name }}": 10, + "{{ cookiecutter.repo_name }}": 4, } OVERVIEW_COUNT = { '"{{ cookiecutter.overview }}",': 0, @@ -53,10 +53,10 @@ } USING_R_YES_COUNT = { "https://github.com/lorenzwalthert/precommit": 2, - "`.lintr`": 1, - "`.Rprofile`": 1, - "`DESCRIPTION`": 2, - "`startup.R`": 1, + "`.lintr`": 0, + "`.Rprofile`": 0, + "`DESCRIPTION`": 0, + "`startup.R`": 0, } @@ -165,7 +165,7 @@ def recursive_open_and_count_search_terms( ( "repo_name", "repo_2", - {**REPO_NAME_COUNT, "{{ cookiecutter.repo_name }}": 11}, + {**REPO_NAME_COUNT, "{{ cookiecutter.repo_name }}": 5}, {"using_R": "Yes"}, ), ("overview", "overview_1", OVERVIEW_COUNT, {"using_R": "No"}), diff --git a/{{ cookiecutter.repo_name }}/.env b/{{ cookiecutter.repo_name }}/.env index 739bda8..c71f7d7 100644 --- a/{{ cookiecutter.repo_name }}/.env +++ b/{{ cookiecutter.repo_name }}/.env @@ -24,30 +24,15 @@ # Add environment variables for the `data` directories DIR_DATA=./data -DIR_DATA_EXTERNAL=./data/external -DIR_DATA_RAW=./data/raw -DIR_DATA_INTERIM=./data/interim -DIR_DATA_PROCESSED=./data/processed # Add environment variables for the `docs` directory DIR_DOCS=./docs -# Add environment variables for the `notebooks` directory -DIR_NOTEBOOKS=./notebooks - -# Add environment variables for the `outputs` directory -DIR_OUTPUTS=./outputs - # Add environment variable for package directory -DIR_COOKIECUTTER.REPO_NAME = ./{{ cookiecutter.repo_name }} +DIR_SRC = ./src # Add environment variables for the `src` directories -DIR_COOKIECUTTER.REPO_NAME_SRC=./{{ cookiecutter.repo_name }}/src -DIR_SRC_MAKE_DATA=./{{ cookiecutter.repo_name }}/src/make_data -DIR_SRC_MAKE_FEATURES=./{{ cookiecutter.repo_name }}/src/make_features -DIR_SRC_MAKE_MODELS=./{{ cookiecutter.repo_name }}/src/make_models -DIR_SRC_MAKE_VISUALISATIONS=./{{ cookiecutter.repo_name }}/src/make_visualisations -DIR_SRC_UTILS=./{{ cookiecutter.repo_name }}/src/utils +DIR_SRC_COOKIECUTTER.REPO_NAME=./src/{{ cookiecutter.repo_name }} # Add environment variables for the `tests` directory -DIR_COOKIECUTTER.REPO_NAME_TESTS=./{{ cookiecutter.repo_name }}/tests +DIR_TESTS=./tests diff --git a/{{ cookiecutter.repo_name }}/data/README.md b/{{ cookiecutter.repo_name }}/data/README.md index 54fbd5a..269f086 100644 --- a/{{ cookiecutter.repo_name }}/data/README.md +++ b/{{ cookiecutter.repo_name }}/data/README.md @@ -1,16 +1,11 @@ # `data` folder overview -Any data that needs to be stored locally should be saved in this location. This folder, -and its sub-folders, are not version-controlled. +Any data that needs to be stored locally should be saved in this location. -The sub-folders should be used as follows: - -- `external`: any data that will not be processed at all, such as reference data; -- `raw`: any raw data before any processing; -- `interim`: any raw data that has been partially processed and, for whatever reason, - needs to be stored before further processing is completed; and -- `processed`: any raw or interim data that has been fully processed into its final - state. +Most data files should not be pushed to your repository, therefore make +sure your .gitignore file excludes your data files from git commits. An +exemption to this may be an example data file that another user may use in +a tutorial for your package. The paths for these directories are loaded as environment variables by the `.envrc` file. To load them in Python, use any or all of the following code: @@ -20,8 +15,4 @@ import os # Load environment variables for the `data` folder, and its sub-folders DIR_DATA = os.getenv("DIR_DATA") -DIR_DATA_EXTERNAL = os.getenv("DIR_DATA_EXTERNAL") -DIR_DATA_RAW = os.getenv("DIR_DATA_RAW") -DIR_DATA_INTERIM = os.getenv("DIR_DATA_INTERIM") -DIR_DATA_PROCESSED = os.getenv("DIR_DATA_PROCESSED") ``` diff --git a/{{ cookiecutter.repo_name }}/data/external/.gitkeep b/{{ cookiecutter.repo_name }}/data/external/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/{{ cookiecutter.repo_name }}/data/interim/.gitkeep b/{{ cookiecutter.repo_name }}/data/interim/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/{{ cookiecutter.repo_name }}/data/processed/.gitkeep b/{{ cookiecutter.repo_name }}/data/processed/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/{{ cookiecutter.repo_name }}/data/raw/.gitkeep b/{{ cookiecutter.repo_name }}/data/raw/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/{{ cookiecutter.repo_name }}/docs/contributor_guide/writing_sphinx_documentation.md b/{{ cookiecutter.repo_name }}/docs/contributor_guide/writing_sphinx_documentation.md index 89d232c..030e31d 100644 --- a/{{ cookiecutter.repo_name }}/docs/contributor_guide/writing_sphinx_documentation.md +++ b/{{ cookiecutter.repo_name }}/docs/contributor_guide/writing_sphinx_documentation.md @@ -58,15 +58,6 @@ create a ReST file, and add the following line to reference the `{{ cookiecutter Then, elsewhere in the body, [call the `autosummary` directive to generate the docstrings as ReST stub files][sphinx-autosummary]. -```rest -.. autosummary:: - :toctree: api/ - - hello - world - -``` - [This will create something similar to the `pandas` API reference][pandas-api-reference]. @@ -95,17 +86,6 @@ create a Markdown file, and add the following line to reference the `{{ cookiecu Then, elsewhere in the body, [call the `autosummary` directive to generate the docstrings as ReST stub files][sphinx-autosummary]. -````md -```{eval-rst} -.. autosummary:: - :toctree: api/ - - hello - world - -``` -```` - ### Including Markdown files outside the `docs` folder [MyST lets you include Markdown files outside the `docs` folder][myst-include]. diff --git a/{{ cookiecutter.repo_name }}/docs/index.md b/{{ cookiecutter.repo_name }}/docs/index.md index 8e4234d..424cf46 100644 --- a/{{ cookiecutter.repo_name }}/docs/index.md +++ b/{{ cookiecutter.repo_name }}/docs/index.md @@ -9,6 +9,4 @@ self ./user_guide/README.md ./contributor_guide/README.md ./aqa/README.md -./structure/README.md -./reference/README.md ``` diff --git a/{{ cookiecutter.repo_name }}/docs/reference/README.md b/{{ cookiecutter.repo_name }}/docs/reference/README.md deleted file mode 100644 index d018b63..0000000 --- a/{{ cookiecutter.repo_name }}/docs/reference/README.md +++ /dev/null @@ -1,14 +0,0 @@ -# `src` API reference - -This page gives an overview of all public `src` objects, functions and methods. All -classes and functions exposed in `src.*` namespace are public. - -```{toctree} -:maxdepth: 2 -./make_data.md -./make_features.md -./make_models.md -./make_visualisations.md -./utils.md -./src.md -``` diff --git a/{{ cookiecutter.repo_name }}/docs/reference/make_data.md b/{{ cookiecutter.repo_name }}/docs/reference/make_data.md deleted file mode 100644 index 4adfdbe..0000000 --- a/{{ cookiecutter.repo_name }}/docs/reference/make_data.md +++ /dev/null @@ -1,15 +0,0 @@ -# Data generation - -These `{{ cookiecutter.repo_name }}` package functions generate data. - -```{eval-rst} -.. currentmodule:: {{ cookiecutter.repo_name }} -``` - -## Function heading - -```{eval-rst} -.. autosummary:: - :toctree: api/ - -``` diff --git a/{{ cookiecutter.repo_name }}/docs/reference/make_features.md b/{{ cookiecutter.repo_name }}/docs/reference/make_features.md deleted file mode 100644 index 7beaef6..0000000 --- a/{{ cookiecutter.repo_name }}/docs/reference/make_features.md +++ /dev/null @@ -1,15 +0,0 @@ -# Feature generation - -These `{{ cookiecutter.repo_name }}` package functions create features. - -```{eval-rst} -.. currentmodule:: {{ cookiecutter.repo_name }} -``` - -## Function heading - -```{eval-rst} -.. autosummary:: - :toctree: api/ - -``` diff --git a/{{ cookiecutter.repo_name }}/docs/reference/make_models.md b/{{ cookiecutter.repo_name }}/docs/reference/make_models.md deleted file mode 100644 index 1a607a4..0000000 --- a/{{ cookiecutter.repo_name }}/docs/reference/make_models.md +++ /dev/null @@ -1,15 +0,0 @@ -# Model generation - -These `{{ cookiecutter.repo_name }}` package functions create models. - -```{eval-rst} -.. currentmodule:: {{ cookiecutter.repo_name }} -``` - -## Function heading - -```{eval-rst} -.. autosummary:: - :toctree: api/ - -``` diff --git a/{{ cookiecutter.repo_name }}/docs/reference/make_visualisations.md b/{{ cookiecutter.repo_name }}/docs/reference/make_visualisations.md deleted file mode 100644 index 72056f8..0000000 --- a/{{ cookiecutter.repo_name }}/docs/reference/make_visualisations.md +++ /dev/null @@ -1,15 +0,0 @@ -# Create visualisations - -These `{{ cookiecutter.repo_name }}` package functions create visualisations. - -```{eval-rst} -.. currentmodule:: {{ cookiecutter.repo_name }} -``` - -## Function heading - -```{eval-rst} -.. autosummary:: - :toctree: api/ - -``` diff --git a/{{ cookiecutter.repo_name }}/docs/reference/src.md b/{{ cookiecutter.repo_name }}/docs/reference/src.md deleted file mode 100644 index a4b4c4f..0000000 --- a/{{ cookiecutter.repo_name }}/docs/reference/src.md +++ /dev/null @@ -1,15 +0,0 @@ -# Package functions - -These are miscellaneous functions in the `{{ cookiecutter.repo_name }}` package. - -```{eval-rst} -.. currentmodule:: {{ cookiecutter.repo_name }} -``` - -## Function heading - -```{eval-rst} -.. autosummary:: - :toctree: api/ - -``` diff --git a/{{ cookiecutter.repo_name }}/docs/reference/utils.md b/{{ cookiecutter.repo_name }}/docs/reference/utils.md deleted file mode 100644 index 2dbfe22..0000000 --- a/{{ cookiecutter.repo_name }}/docs/reference/utils.md +++ /dev/null @@ -1,15 +0,0 @@ -# Create utility functions - -These `{{ cookiecutter.repo_name }}` package functions create utility functions. - -```{eval-rst} -.. currentmodule:: {{ cookiecutter.repo_name }} -``` - -## Function heading - -```{eval-rst} -.. autosummary:: - :toctree: api/ - -``` diff --git a/{{ cookiecutter.repo_name }}/docs/structure/README.md b/{{ cookiecutter.repo_name }}/docs/structure/README.md deleted file mode 100644 index 4a7158b..0000000 --- a/{{ cookiecutter.repo_name }}/docs/structure/README.md +++ /dev/null @@ -1,203 +0,0 @@ -# `{{ cookiecutter.repo_name }}` structure - -This page provides information on the repository's structure. The repository's folder -structure is explained here: - -```{toctree} -:maxdepth: 2 -./data.md -./docs.md -./notebooks.md -./outputs.md -./src.md -./tests.md -``` - -## Top-level files - -Each subsection here contains a brief description about the files at the top-level of -this Git repository. - -### `.env` - -Environment variables go here, and can be read in by Python using the `python-dotenv` package, and `os.getenv`: - -```in script -from dotenv import load_dotenv -import os - -#Load the environment variables from the `.env` file, overriding any system -#environment variables -load_dotenv(override=True) - -# Example variable -EXAMPLE_VARIABLE = os.getenv("EXAMPLE_VARIABLE") -``` - -### `.envrc` - -Orchestration file to load environment variables from the `.env` and `.secrets` files. -Only used by systems with `direnv` (https://direnv.net/) installed. Environment -variables can be read in by Python using `os.getenv` _without_ using `python-dotenv`: - -```in script -import os - -# Example variable -EXAMPLE_VARIABLE = os.getenv("EXAMPLE_VARIABLE") -``` - - -### `.flake8` - -A configuration file for the `flake8` Python package that provides linting. This file -is based on the [common configuration described in the GDS Way][gds-way-flake8]. - -### `.gitignore` - -A `.gitignore` file to ignore certain files and folders from this Git repository. [See -the contributor guide to modift the `.gitignore` file][docs-updating-gitignore]. - -{% if cookiecutter.using_R == "Yes" -%} -### `.lintr` - -R-specific configuration file for styling R code. - -{% endif -%} - -### `.pre-commit-config.yaml` - -[A pre-commit hook configuration file][docs-pre-commit-hooks]. - -{% if cookiecutter.using_R == "Yes" -%} -### `.Rprofile` - -R-specific initialisation file that runs automatically when starting R. - -{% endif -%} - -### `.secrets` - -A file to store all secrets and credentials as environment variables. [This file is -read-in by `.envrc`](#envrc), when [loading environment variables with the `direnv` -shell extension][direnv], but is not tracked by Git. - -### `.secrets.baseline` - -[Baseline file for the `detect-secrets` to detect secrets][detect-secrets]. In -conjunction with `pre-commit`, `detect-secrets` prevents secrets from being committed -to the repository. The baseline file flags secret-like data that the user deliberately -wishes to commit the to repository. - -### `CODE_OF_CONDUCT.md` - -[The Code of Conduct for contributors to this project][code-of-conduct], including -maintainers and `{{ cookiecutter.organisation_handle }}` organisation owners. - -### `conftest.py` - -File to contain shared fixture functions for the `pytest` tests in the `tests` folder. - -### `CONTRIBUTING.md` - -The contributing guidelines for this project. - -{% if cookiecutter.using_R == "Yes" -%} -### `DESCRIPTION` - -R-specific information related to the project including the name, authors and packages -necessary for the project. - -{% endif -%} - -### `LICENSE` - -The licence for this project. Unless stated otherwise, the codebase is released under -the MIT License. This covers both the codebase and any sample code in the -documentation. The documentation is © Crown copyright and available under the terms of -the Open Government 3.0 licence. - -### `Makefile` - -The `Makefile` contains a set of commands for the `make` utility. Run the `help` -command for further information at the top-level of the Git repository. - -```shell -make help -``` - -### `pyproject.toml` - -A file containing Python project settings. This includes configuration settings for: - -- [`isort`](#isort) -- [`pytest`](#pytest) -- [code coverage](#code-coverage) - -#### `isort` - -Python imports are arranged according to the [specification defined by `black`][black]. - -#### `pytest` - -To run the tests within the `tests` folder using the `pytest` Python package, enter -the following command: - -```shell -pytest -``` - -#### Code coverage - -To run code coverage using the `coverage` Python package with `pytest`, enter the -following command: - -```shell -coverage run -m pytest -coverage html -``` - -or using the `make` command: - -```shell -make coverage_html -``` - -A code coverage report in HTML will be produced on the code in the `src` folder. This -HTML report can be accessed at `htmlcov/index.html`. - -### `README.md` - -An overview of the Git repository, including all necessary instructions to run the code. - -### `requirements.txt` - -A list of Python package requirements for this Git repository, which can be installed -using the `pip install` command. - -```shell -pip install --requirement requirements.txt -``` - -Alternatively, to install the requirements file along with pre-commit hooks, run the -following command: - -```shell -make requirements -``` - -{% if cookiecutter.using_R == "Yes" -%} -### `startup.R` - -R-specific file that installs necessary packages specified in the `DESCRIPTION` file -upon starting R using `.Rprofile`. - -{% endif -%} - -[black]: https://black.readthedocs.io/en/stable/ -[code-of-conduct]:https://github.com/best-practice-and-impact/govcookiecutter/blob/main/%7B%7B%20cookiecutter.repo_name%20%7D%7D/docs/contributor_guide/CODE_OF_CONDUCT.md -[detect-secrets]: https://github.com/Yelp/detect-secrets -[direnv]: https://direnv.net/ -[docs-pre-commit-hooks]: https://github.com/best-practice-and-impact/govcookiecutter/blob/main/%7B%7B%20cookiecutter.repo_name%20%7D%7D/docs/contributor_guide/pre_commit_hooks.md -[docs-updating-gitignore]: https://github.com/best-practice-and-impact/govcookiecutter/blob/main/%7B%7B%20cookiecutter.repo_name%20%7D%7D/docs/contributor_guide/updating_gitignore.md -[gds-way-flake8]: https://gds-way.cloudapps.digital/manuals/programming-languages/python/python.html#common-configuration diff --git a/{{ cookiecutter.repo_name }}/docs/structure/data.md b/{{ cookiecutter.repo_name }}/docs/structure/data.md deleted file mode 100644 index f06a7ee..0000000 --- a/{{ cookiecutter.repo_name }}/docs/structure/data.md +++ /dev/null @@ -1,3 +0,0 @@ -```{include} ../../data/README.md -:relative-docs: ../docs/structure -``` diff --git a/{{ cookiecutter.repo_name }}/docs/structure/docs.md b/{{ cookiecutter.repo_name }}/docs/structure/docs.md deleted file mode 100644 index b95ce84..0000000 --- a/{{ cookiecutter.repo_name }}/docs/structure/docs.md +++ /dev/null @@ -1,3 +0,0 @@ -```{include} ../README.md -:relative-docs: ../docs -``` diff --git a/{{ cookiecutter.repo_name }}/docs/structure/notebooks.md b/{{ cookiecutter.repo_name }}/docs/structure/notebooks.md deleted file mode 100644 index b950f2c..0000000 --- a/{{ cookiecutter.repo_name }}/docs/structure/notebooks.md +++ /dev/null @@ -1,3 +0,0 @@ -```{include} ../../notebooks/README.md -:relative-docs: ../docs/structure -``` diff --git a/{{ cookiecutter.repo_name }}/docs/structure/outputs.md b/{{ cookiecutter.repo_name }}/docs/structure/outputs.md deleted file mode 100644 index f15500f..0000000 --- a/{{ cookiecutter.repo_name }}/docs/structure/outputs.md +++ /dev/null @@ -1,3 +0,0 @@ -```{include} ../../outputs/README.md -:relative-docs: ../docs/structure -``` diff --git a/{{ cookiecutter.repo_name }}/docs/structure/{{ cookiecutter.repo_name }}.md b/{{ cookiecutter.repo_name }}/docs/structure/{{ cookiecutter.repo_name }}.md deleted file mode 100644 index 6472d67..0000000 --- a/{{ cookiecutter.repo_name }}/docs/structure/{{ cookiecutter.repo_name }}.md +++ /dev/null @@ -1,3 +0,0 @@ -```{include} ../../{{ cookiecutter.repo_name }}/README.md -:relative-docs: ../docs/structure -``` diff --git a/{{ cookiecutter.repo_name }}/notebooks/README.md b/{{ cookiecutter.repo_name }}/notebooks/README.md deleted file mode 100644 index b657204..0000000 --- a/{{ cookiecutter.repo_name }}/notebooks/README.md +++ /dev/null @@ -1,7 +0,0 @@ -# `notebooks` folder overview - -All Jupyter notebooks should be stored in this folder. - -The `.envrc` file should automatically add the entire project path into the -`PYTHONPATH` environment variable. This should allow you to directly import `src` in -your notebook. diff --git a/{{ cookiecutter.repo_name }}/outputs/README.md b/{{ cookiecutter.repo_name }}/outputs/README.md deleted file mode 100644 index 74fb5bc..0000000 --- a/{{ cookiecutter.repo_name }}/outputs/README.md +++ /dev/null @@ -1,12 +0,0 @@ -# `outputs` folder overview - -All outputs from the project should be stored here. This folder path for these -directories is loaded as an environment variable by the `.envrc` file; to load them in -Python, use the following code: - -```python -import os - -# Load environment variables for the `outputs` folder -DIR_OUTPUTS = os.getenv("DIR_OUTPUTS") -``` diff --git a/{{ cookiecutter.repo_name }}/pyproject.toml b/{{ cookiecutter.repo_name }}/pyproject.toml index fb0484a..822946b 100644 --- a/{{ cookiecutter.repo_name }}/pyproject.toml +++ b/{{ cookiecutter.repo_name }}/pyproject.toml @@ -21,5 +21,5 @@ addopts = [ ] doctest_optionflags = "NORMALIZE_WHITESPACE" testpaths = [ - "./{{ cookiecutter.repo_name }}/tests" + "./tests" ] diff --git a/{{ cookiecutter.repo_name }}/setup.cfg b/{{ cookiecutter.repo_name }}/setup.cfg index dd957f6..1201da4 100644 --- a/{{ cookiecutter.repo_name }}/setup.cfg +++ b/{{ cookiecutter.repo_name }}/setup.cfg @@ -23,5 +23,6 @@ install_requires = Sphinx toml python_requires = >=3.6 - +package_dir = + =./src zip_safe = no diff --git a/{{ cookiecutter.repo_name }}/{{ cookiecutter.repo_name }}/__init__.py b/{{ cookiecutter.repo_name }}/src/__init__.py similarity index 100% rename from {{ cookiecutter.repo_name }}/{{ cookiecutter.repo_name }}/__init__.py rename to {{ cookiecutter.repo_name }}/src/__init__.py diff --git a/{{ cookiecutter.repo_name }}/{{ cookiecutter.repo_name }}/src/__init__.py b/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.repo_name }}/__init__.py similarity index 100% rename from {{ cookiecutter.repo_name }}/{{ cookiecutter.repo_name }}/src/__init__.py rename to {{ cookiecutter.repo_name }}/src/{{ cookiecutter.repo_name }}/__init__.py diff --git a/{{ cookiecutter.repo_name }}/{{ cookiecutter.repo_name }}/src/make_data/__init__.py b/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.repo_name }}/utils/__init__.py similarity index 100% rename from {{ cookiecutter.repo_name }}/{{ cookiecutter.repo_name }}/src/make_data/__init__.py rename to {{ cookiecutter.repo_name }}/src/{{ cookiecutter.repo_name }}/utils/__init__.py diff --git a/{{ cookiecutter.repo_name }}/{{ cookiecutter.repo_name }}/tests/README.md b/{{ cookiecutter.repo_name }}/tests/README.md similarity index 100% rename from {{ cookiecutter.repo_name }}/{{ cookiecutter.repo_name }}/tests/README.md rename to {{ cookiecutter.repo_name }}/tests/README.md diff --git a/{{ cookiecutter.repo_name }}/{{ cookiecutter.repo_name }}/src/make_features/__init__.py b/{{ cookiecutter.repo_name }}/tests/__init__.py similarity index 100% rename from {{ cookiecutter.repo_name }}/{{ cookiecutter.repo_name }}/src/make_features/__init__.py rename to {{ cookiecutter.repo_name }}/tests/__init__.py diff --git a/{{ cookiecutter.repo_name }}/{{ cookiecutter.repo_name }}/src/README.md b/{{ cookiecutter.repo_name }}/{{ cookiecutter.repo_name }}/src/README.md deleted file mode 100644 index e737ef5..0000000 --- a/{{ cookiecutter.repo_name }}/{{ cookiecutter.repo_name }}/src/README.md +++ /dev/null @@ -1,24 +0,0 @@ -# `src` package overview - -All functions for this project, should be stored in this folder. All tests should be -stored in the `tests` folder, which is one-level above this folder in the main project -directory. - -The sub-folders should be used as follows: - -- `make_data`: data processing-related functions -- `make_features`: feature-related functions, for example, functions to create features - from processed data -- `make_models`: model-related functions -- `make_visualisations`: functions to produce visualisations -- `utils`: utility functions that are helpful in the project - -Feel free to create/rename/delete these folders as required, as they will not be -necessary for each and every project. - -It is strongly suggested that you import functions in the `src/__init__.py` script. You -should also try to use absolute imports in this script whenever possible. Relative -imports are not discouraged, but can be an issue for projects where the directory -structure is likely to change. See [PEP 328 for details on absolute imports][pep-328]. - -[pep-328]: https://www.python.org/dev/peps/pep-0328/ diff --git a/{{ cookiecutter.repo_name }}/{{ cookiecutter.repo_name }}/src/make_models/__init__.py b/{{ cookiecutter.repo_name }}/{{ cookiecutter.repo_name }}/src/make_models/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/{{ cookiecutter.repo_name }}/{{ cookiecutter.repo_name }}/src/make_visualisations/__init__.py b/{{ cookiecutter.repo_name }}/{{ cookiecutter.repo_name }}/src/make_visualisations/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/{{ cookiecutter.repo_name }}/{{ cookiecutter.repo_name }}/src/utils/__init__.py b/{{ cookiecutter.repo_name }}/{{ cookiecutter.repo_name }}/src/utils/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/{{ cookiecutter.repo_name }}/{{ cookiecutter.repo_name }}/tests/__init__.py b/{{ cookiecutter.repo_name }}/{{ cookiecutter.repo_name }}/tests/__init__.py deleted file mode 100644 index e69de29..0000000 From 308c7bd740927a62c328d3c51147fd3a1f3e6a91 Mon Sep 17 00:00:00 2001 From: joshlynchONS <118733244+joshlynchONS@users.noreply.github.com> Date: Mon, 6 Mar 2023 13:32:42 +0000 Subject: [PATCH 03/13] docs: fix broken makefile link --- {{ cookiecutter.repo_name }}/docs/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/{{ cookiecutter.repo_name }}/docs/README.md b/{{ cookiecutter.repo_name }}/docs/README.md index 1974dee..7011656 100644 --- a/{{ cookiecutter.repo_name }}/docs/README.md +++ b/{{ cookiecutter.repo_name }}/docs/README.md @@ -28,5 +28,5 @@ These files document how this project meets organisational [guidance on producin quality analysis for HM Government projects][aqua-book]. [aqua-book]: https://www.gov.uk/government/publications/the-aqua-book-guidance-on-producing-quality-analysis-for-government -[docs-makefile]: https://github.com/best-practice-and-impact/govcookiecutter/blob/main/%7B%7B%20cookiecutter.repo_name%20%7D%7D/docs/structure/README.md#makefile +[docs-makefile]: https://github.com/best-practice-and-impact/govcookiecutter/blob/main/%7B%7B%20cookiecutter.repo_name%20%7D%7D/Makefile [writing-sphinx-documentation]: https://github.com/best-practice-and-impact/govcookiecutter/blob/main/%7B%7B%20cookiecutter.repo_name%20%7D%7D/docs/contributor_guide/writing_sphinx_documentation.md From 548dabda30fec5d5c6a05396e1d13b7ff4326fdb Mon Sep 17 00:00:00 2001 From: joshlynchONS <118733244+joshlynchONS@users.noreply.github.com> Date: Wed, 8 Mar 2023 14:28:28 +0000 Subject: [PATCH 04/13] Add: example test and module * fix: remove data sub folders and update env variables * fix: remove unnecessary root folders and update environment variables * fix: rearrange package and src and update env variables * fix: fix test env variable * fix: update setup files for install * fix: add example test to env variables * Revert "fix: add example test to env variables" This reverts commit eb306638fde955664ee2ad2b68ecd7956516cc66. * Revert "fix: update setup files for install" This reverts commit 1021857ab36e1e1727e683af229b3204bf0b256d. * Revert "fix: fix test env variable" This reverts commit 27745eeda354861ed15bbdcc79834562e1177be0. * Revert "fix: rearrange package and src and update env variables" This reverts commit 7124468ee404c2dd992ce0ff3b072db82d4cb78b. * fix: move test folder and change test env variable * fix: remove module folders from package and update env variables * fix: swap src and package folders around and update env variables * fix: remove utils folder env variable * docs: remove structure documentation * fix: update injected variables for structure docs removal * docs: remove reference docs for package * fix: update injected variables for reference folder removal * fix: update repo name count in injected variables * docs: update docs to reflect the removed folders * fix: remove autosummary code blocks in writing sphinx docs * fix: updated package location for pip install * fix: update repo injected variables * add: add demo files for pytest * fix: update pre-commit yml to exclude demo test * fix: add ignore test to github workflow * fix: ignore demo pytest file * fix: add to pyproject exclude demo pytest * fix: eclude end user package tests from cookiecutter test * fix: update env variable for example test module * fix: add ignore file to workflow and revert last 2 commits * fix: spelling mistake in workflow * fix: add omit pytest demo to pyproject * fix: exclude pytest demo from coverage * fix: update pyproject * fix: update pyproject * fix: update pyproject * fix: update pyproject * fix: update pyproject * fix: remove unnecessary pyproject lines * fix: remove unnecessary code from workflow * fix: fix directory for setup.cfg * fix: update injected variable counts * chore: refactor code in example test to match black * docs: add docs for pytest * fix: update test to test file per module * fix: replace spaces and '-' in repo name for _ to stop error in python imports * fix: fix broken makefile link * fix: replace spaces and -'s with _'s in python import * fix: ammend package name in setup docs and rename package in folders * fix: update package directory in env * fix: remove extra folders from merge --- .pre-commit-config.yaml | 2 + cookiecutter.json | 2 +- pyproject.toml | 3 +- {{ cookiecutter.repo_name }}/.env | 2 +- .../docs/user_guide/using_pytest.md | 40 +++++++++++++++++++ {{ cookiecutter.repo_name }}/setup.cfg | 2 +- .../__init__.py | 0 .../utils/__init__.py | 0 .../utils/example_module.py | 6 +++ .../tests/test_example_module.py | 11 +++++ 10 files changed, 63 insertions(+), 5 deletions(-) create mode 100644 {{ cookiecutter.repo_name }}/docs/user_guide/using_pytest.md rename {{ cookiecutter.repo_name }}/src/{{{ cookiecutter.repo_name }} => {{ cookiecutter.repo_name.lower().replace(' ', '_').replace('-', '_') }}}/__init__.py (100%) rename {{ cookiecutter.repo_name }}/src/{{{ cookiecutter.repo_name }} => {{ cookiecutter.repo_name.lower().replace(' ', '_').replace('-', '_') }}}/utils/__init__.py (100%) create mode 100644 {{ cookiecutter.repo_name }}/src/{{ cookiecutter.repo_name.lower().replace(' ', '_').replace('-', '_') }}/utils/example_module.py create mode 100644 {{ cookiecutter.repo_name }}/tests/test_example_module.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d292129..ab20e3a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -24,11 +24,13 @@ repos: - id: black name: black - consistent Python code formatting (auto-fixes) language_version: python # Should be a command that runs python3.6+ + exclude: test_example_module.py - repo: https://github.com/PyCQA/flake8 rev: 5.0.4 hooks: - id: flake8 name: flake8 - Python linting + exclude: test_example_module.py - repo: https://github.com/Yelp/detect-secrets rev: v1.0.3 hooks: diff --git a/cookiecutter.json b/cookiecutter.json index c8d9937..af0116e 100644 --- a/cookiecutter.json +++ b/cookiecutter.json @@ -8,7 +8,7 @@ "organisational_framework": ["GDS", "N/A"], "project_name": "Your new project name", - "repo_name": "{{ cookiecutter.project_name.lower().replace(' ', '-') }}", + "repo_name": "{{ cookiecutter.project_name.lower().replace(' ', '_').replace('-', '_') }}", "overview": "Brief overview of your project.", "project_version": "0.0.1", diff --git a/pyproject.toml b/pyproject.toml index 52b9130..a018f0d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,6 +25,5 @@ addopts = [ ] doctest_optionflags = "NORMALIZE_WHITESPACE" testpaths = [ - "./tests", - "./{{ cookiecutter.repo_name }}/tests" + "./tests" ] diff --git a/{{ cookiecutter.repo_name }}/.env b/{{ cookiecutter.repo_name }}/.env index c71f7d7..f94a3cc 100644 --- a/{{ cookiecutter.repo_name }}/.env +++ b/{{ cookiecutter.repo_name }}/.env @@ -32,7 +32,7 @@ DIR_DOCS=./docs DIR_SRC = ./src # Add environment variables for the `src` directories -DIR_SRC_COOKIECUTTER.REPO_NAME=./src/{{ cookiecutter.repo_name }} +DIR_SRC_COOKIECUTTER.REPO_NAME.LOWER().REPLACE('','_').REPLACE('-','_') =./src/{{ cookiecutter.repo_name.lower().replace(' ', '_').replace('-', '_') }} # Add environment variables for the `tests` directory DIR_TESTS=./tests diff --git a/{{ cookiecutter.repo_name }}/docs/user_guide/using_pytest.md b/{{ cookiecutter.repo_name }}/docs/user_guide/using_pytest.md new file mode 100644 index 0000000..e7b2fed --- /dev/null +++ b/{{ cookiecutter.repo_name }}/docs/user_guide/using_pytest.md @@ -0,0 +1,40 @@ +# Using pytest + +We use [`pytest`][pytest] to create and run all of our python based testing. Pytest is the most commonly used python module for testing python code. Testing your code is vital for followling coding best practices and has numerous benefits such as: + +* They help to debug your code +* They make your write more efficient code first time +* They make you think about what precisely your code is doing as you right it +* They provide a sort of documentation for your code +* They help to keep your deployment smooth. + +## Structure + +There should be a `tests` folder in the root directory of your repository containing all the tests that relate to your package. It sits outside of your package because users that want to just use your package will not necessarily need the tests. The tests are there for contributers to use, and if they are contributing they will clone the whole repository, not just the package. + +There is an example pytest folder structure and file in your package that demonstrates this structure. + +## Writing pytests + +For pytest to find your tests, all test files and tests must either start with `test_` or finish with `_test.py` + +The `test_example_module.py` example test file provides an example of these restrictions. + +## Running pytest +### In the terminal + +There are a few ways to run pytests in your terminal. The easiest is by running +```shell +pytest +``` +in your root directory. This will find any existing pytests within your directory and run them. + +If you only want to run pytests in a specific pytest file you can run +```shell +pytest tests/test_example_module.py +``` + +You can try both of these in the root directory of your new repository. + + +[pytest]: https://pypi.org/project/pytest/ diff --git a/{{ cookiecutter.repo_name }}/setup.cfg b/{{ cookiecutter.repo_name }}/setup.cfg index 1201da4..f64726c 100644 --- a/{{ cookiecutter.repo_name }}/setup.cfg +++ b/{{ cookiecutter.repo_name }}/setup.cfg @@ -11,7 +11,7 @@ classifiers = [options] packages = - {{ cookiecutter.repo_name }} + {{ cookiecutter.repo_name.lower().replace(' ', '_').replace('-', '_') }} install_requires = coverage detect-secrets == 1.0.3 diff --git a/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.repo_name }}/__init__.py b/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.repo_name.lower().replace(' ', '_').replace('-', '_') }}/__init__.py similarity index 100% rename from {{ cookiecutter.repo_name }}/src/{{ cookiecutter.repo_name }}/__init__.py rename to {{ cookiecutter.repo_name }}/src/{{ cookiecutter.repo_name.lower().replace(' ', '_').replace('-', '_') }}/__init__.py diff --git a/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.repo_name }}/utils/__init__.py b/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.repo_name.lower().replace(' ', '_').replace('-', '_') }}/utils/__init__.py similarity index 100% rename from {{ cookiecutter.repo_name }}/src/{{ cookiecutter.repo_name }}/utils/__init__.py rename to {{ cookiecutter.repo_name }}/src/{{ cookiecutter.repo_name.lower().replace(' ', '_').replace('-', '_') }}/utils/__init__.py diff --git a/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.repo_name.lower().replace(' ', '_').replace('-', '_') }}/utils/example_module.py b/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.repo_name.lower().replace(' ', '_').replace('-', '_') }}/utils/example_module.py new file mode 100644 index 0000000..4f86b44 --- /dev/null +++ b/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.repo_name.lower().replace(' ', '_').replace('-', '_') }}/utils/example_module.py @@ -0,0 +1,6 @@ +def hello_world(name): + if isinstance(name, str): + string = "hello world and hello " + name + return string + else: + raise TypeError("The 'name' variable entered was not a string") diff --git a/{{ cookiecutter.repo_name }}/tests/test_example_module.py b/{{ cookiecutter.repo_name }}/tests/test_example_module.py new file mode 100644 index 0000000..708f858 --- /dev/null +++ b/{{ cookiecutter.repo_name }}/tests/test_example_module.py @@ -0,0 +1,11 @@ +from {{ cookiecutter.repo_name.lower().replace(' ', '_').replace('-', '_') }}.utils.example_module import hello_world +import pytest + + +class TestHelloWorld: + def test_string_concat(self): + assert hello_world("ONS") == "hello world and hello ONS" + + def test_type_error(self): + with pytest.raises(TypeError): + hello_world(1) From 07c1d89b856ce3049e8149e81f228eb115ed5bcc Mon Sep 17 00:00:00 2001 From: joshlynchONS <118733244+joshlynchONS@users.noreply.github.com> Date: Thu, 23 Mar 2023 15:24:37 +0000 Subject: [PATCH 05/13] Remove: Remove data folder * remove: remove data folder * fix: fix data folder env variable and data within the readme --- README.md | 2 +- {{ cookiecutter.repo_name }}/.env | 4 ---- {{ cookiecutter.repo_name }}/data/README.md | 18 ------------------ 3 files changed, 1 insertion(+), 23 deletions(-) delete mode 100644 {{ cookiecutter.repo_name }}/data/README.md diff --git a/README.md b/README.md index 15d7421..bb61781 100644 --- a/README.md +++ b/README.md @@ -134,7 +134,7 @@ guidelines](https://github.com/best-practice-and-impact/govcookiecutter/blob/mai ## Acknowledgements [This template is based off the DrivenData Cookiecutter Data Science -project][drivendata]. Specifically, it uses similar `data` and `src` folder structures, +project][drivendata]. Specifically, it uses a similar `src` folder structure, and a modified version of the `help` commands in the `Makefile`s. [aqua-book]: https://www.gov.uk/government/publications/the-aqua-book-guidance-on-producing-quality-analysis-for-government diff --git a/{{ cookiecutter.repo_name }}/.env b/{{ cookiecutter.repo_name }}/.env index f94a3cc..76551a3 100644 --- a/{{ cookiecutter.repo_name }}/.env +++ b/{{ cookiecutter.repo_name }}/.env @@ -21,10 +21,6 @@ # DO NOT STORE SECRETS HERE - this file is version-controlled! You should store secrets # in the untracked `.secrets` file. - -# Add environment variables for the `data` directories -DIR_DATA=./data - # Add environment variables for the `docs` directory DIR_DOCS=./docs diff --git a/{{ cookiecutter.repo_name }}/data/README.md b/{{ cookiecutter.repo_name }}/data/README.md deleted file mode 100644 index 269f086..0000000 --- a/{{ cookiecutter.repo_name }}/data/README.md +++ /dev/null @@ -1,18 +0,0 @@ -# `data` folder overview - -Any data that needs to be stored locally should be saved in this location. - -Most data files should not be pushed to your repository, therefore make -sure your .gitignore file excludes your data files from git commits. An -exemption to this may be an example data file that another user may use in -a tutorial for your package. - -The paths for these directories are loaded as environment variables by the -`.envrc` file. To load them in Python, use any or all of the following code: - -```python -import os - -# Load environment variables for the `data` folder, and its sub-folders -DIR_DATA = os.getenv("DIR_DATA") -``` From e9d0727d179b3b123b8529975ea02ebe0aa01d83 Mon Sep 17 00:00:00 2001 From: joshlynchONS <118733244+joshlynchONS@users.noreply.github.com> Date: Tue, 28 Mar 2023 15:23:05 +0100 Subject: [PATCH 06/13] add: Example pipeline * chore: update utils folder name to example modules * fix: update test import statement * add: add more example functions and tests, update exclude files in pre-commit, add run pipeline file * fix: add full path to file to file exlcude in black pre-commit * fix: revert back to old pre-commit yml * fix: excluding run pipeline from tests * fix: fix exlude files in pre-commit * fix: fix flake8 exclude config * fix: update pipeline to run through console using main * fix: omit run pipeline from coverage report * fix: typo in pyproject.toml * fix: fix syntax issues in example files * docs: update package readme to include pipeline * fix: add doc strings to functions * fix: fix pre-commit issue in doc string of run_pipeline * fix: fix doc string linting in run pipeline * chore: update code to fit isort pre-commit --- .pre-commit-config.yaml | 4 +- pyproject.toml | 5 + {{ cookiecutter.repo_name }}/README.md | 57 +++++++----- .../example_config.yml | 3 + .../{utils => example_modules}/__init__.py | 0 .../example_modules/example_module.py | 92 +++++++++++++++++++ .../run_pipeline.py | 25 +++++ .../utils/example_module.py | 6 -- .../tests/test_example_module.py | 49 +++++++++- 9 files changed, 208 insertions(+), 33 deletions(-) create mode 100644 {{ cookiecutter.repo_name }}/src/{{ cookiecutter.repo_name.lower().replace(' ', '_').replace('-', '_') }}/example_config.yml rename {{ cookiecutter.repo_name }}/src/{{ cookiecutter.repo_name.lower().replace(' ', '_').replace('-', '_') }}/{utils => example_modules}/__init__.py (100%) create mode 100644 {{ cookiecutter.repo_name }}/src/{{ cookiecutter.repo_name.lower().replace(' ', '_').replace('-', '_') }}/example_modules/example_module.py create mode 100644 {{ cookiecutter.repo_name }}/src/{{ cookiecutter.repo_name.lower().replace(' ', '_').replace('-', '_') }}/run_pipeline.py delete mode 100644 {{ cookiecutter.repo_name }}/src/{{ cookiecutter.repo_name.lower().replace(' ', '_').replace('-', '_') }}/utils/example_module.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ab20e3a..fdb68b6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -24,13 +24,13 @@ repos: - id: black name: black - consistent Python code formatting (auto-fixes) language_version: python # Should be a command that runs python3.6+ - exclude: test_example_module.py + exclude: test_example_module.py|run_pipeline.py - repo: https://github.com/PyCQA/flake8 rev: 5.0.4 hooks: - id: flake8 name: flake8 - Python linting - exclude: test_example_module.py + exclude: test_example_module.py|run_pipeline.py - repo: https://github.com/Yelp/detect-secrets rev: v1.0.3 hooks: diff --git a/pyproject.toml b/pyproject.toml index a018f0d..c0c350f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,6 +4,11 @@ source = [ "./hooks", "./{{ cookiecutter.repo_name }}/src" ] +omit = [ + "./{{ cookiecutter.repo_name }}/src/{{ cookiecutter.repo_name.lower().replace(' ', '_').replace('-', '_') }}/run_pipeline.py" +] + +[tool.coverage.omit] [tool.coverage.report] exclude_lines = [ diff --git a/{{ cookiecutter.repo_name }}/README.md b/{{ cookiecutter.repo_name }}/README.md index 9836f6a..e87a172 100644 --- a/{{ cookiecutter.repo_name }}/README.md +++ b/{{ cookiecutter.repo_name }}/README.md @@ -12,41 +12,36 @@ located. To start using this project, [first make sure your system meets its requirements](#requirements). -To be added. +It's suggested that you install this pack and it's requirements within a virtual environment. -### Requirements +## Installing the package (Python Only) -[```Contributors have some additional requirements!```][contributing] - -- Python 3.6.1+ installed -{% if cookiecutter.using_R == "Yes" -%} -- R 4.0.4+ installed -{% endif -%} -- a `.secrets` file with the [required secrets and - credentials](#required-secrets-and-credentials) -- [load environment variables][docs-loading-environment-variables] from `.env` - -To install the Python requirements, open your terminal and enter: - -```shell -pip install -r requirements.txt -``` - -## Installing your package (Python Only) - -Whilst in the root folder, in the command prompt, you can install your package using: +Whilst in the root folder, in the command prompt, you can install the package and it's dependencies +using: ```shell pip install -e . ``` -This installs an editable version of your package. Meaning, when you update your +This installs an editable version of the package. Meaning, when you update the package code, you do not have to reinstall it for the changes to take effect. (This saves a lot of time when you test your code) Remember to update the setup and requirement files inline with any changes to your package. The inital files contain the bare minimum to get you started. +## Running the pipeline (Python only) + +The entry point for the pipeline is stored within the package and called `run_pipeline.py`. +To run the pipeline, run the following code in the terminal (whilst in the root directory of the +project). + +```shell +python src/{{ cookiecutter.repo_name.lower().replace(' ', '_').replace('-', '_') }}/run_pipeline.py +``` + +Alternatively, most Python IDE's allow you to run the code directly from the IDE using a `run` button. + ## Required secrets and credentials To run this project, [you need a `.secrets` file with secrets/credentials as @@ -72,6 +67,24 @@ Crown copyright and available under the terms of the Open Government 3.0 licence [If you want to help us build, and improve `{{ cookiecutter.repo_name }}`, view our contributing guidelines][contributing]. +### Requirements + +[```Contributors have some additional requirements!```][contributing] + +- Python 3.6.1+ installed +{% if cookiecutter.using_R == "Yes" -%} +- R 4.0.4+ installed +{% endif -%} +- a `.secrets` file with the [required secrets and + credentials](#required-secrets-and-credentials) +- [load environment variables][docs-loading-environment-variables] from `.env` + +To install the Python requirements, open your terminal and enter: + +```shell +pip install -r requirements.txt +``` + ## Acknowledgements [This project structure is based on the `govcookiecutter` template diff --git a/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.repo_name.lower().replace(' ', '_').replace('-', '_') }}/example_config.yml b/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.repo_name.lower().replace(' ', '_').replace('-', '_') }}/example_config.yml new file mode 100644 index 0000000..7acc3b8 --- /dev/null +++ b/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.repo_name.lower().replace(' ', '_').replace('-', '_') }}/example_config.yml @@ -0,0 +1,3 @@ +user_name: "John Stat" +company: "ONS" +favourite_number: 42 diff --git a/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.repo_name.lower().replace(' ', '_').replace('-', '_') }}/utils/__init__.py b/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.repo_name.lower().replace(' ', '_').replace('-', '_') }}/example_modules/__init__.py similarity index 100% rename from {{ cookiecutter.repo_name }}/src/{{ cookiecutter.repo_name.lower().replace(' ', '_').replace('-', '_') }}/utils/__init__.py rename to {{ cookiecutter.repo_name }}/src/{{ cookiecutter.repo_name.lower().replace(' ', '_').replace('-', '_') }}/example_modules/__init__.py diff --git a/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.repo_name.lower().replace(' ', '_').replace('-', '_') }}/example_modules/example_module.py b/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.repo_name.lower().replace(' ', '_').replace('-', '_') }}/example_modules/example_module.py new file mode 100644 index 0000000..7ec4165 --- /dev/null +++ b/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.repo_name.lower().replace(' ', '_').replace('-', '_') }}/example_modules/example_module.py @@ -0,0 +1,92 @@ +def hello_world(name, company): + """This function returns a welcoming string given and company + and a persons name. + + Parameters + ---------- + name : str + The name you would like to say hello to + company : str + The company name to say hello to + + Returns + ------- + str + Complete string greating a person and the company + + Raises + ------ + TypeError + name is not a str + TypeError + company is not a str + TypeError + neither company or name are str variables + """ + if isinstance(name, str) and isinstance(company, str): + string = "Hello " + name + " and hello everyone at " + company + return string + + elif not isinstance(name, str): + raise TypeError("The 'name' variable entered was not a string") + + elif not isinstance(company, str): + raise TypeError("The 'company' variable entered was not a string") + + else: + raise TypeError("The 'name' and 'company' variables entered were not strings") + + +def print_string(string): + """function that prints a string after validating the input + + Parameters + ---------- + string : str + string to print ouy + + Raises + ------ + TypeError + If 'string' variable is not a str + """ + if isinstance(string, str): + print(string) + else: + raise TypeError("The 'string' variable entered was not a string") + + +def print_favourite_number(favourite_number, name): + """Function to print sentence of a name and a person favourite number + + Parameters + ---------- + favourite_number : int + Integer number to print + name : str + Person name to print + + Raises + ------ + TypeError + The name variable is not a str + TypeError + The favourite_number variable is not an int + TypeError + Both the name and favourite_number varibles are not str and int + respectively. + """ + if isinstance(name, str) and isinstance(favourite_number, int): + string = name + "'s favourite number is " + str(favourite_number) + "!" + print(string) + + elif not isinstance(name, str): + raise TypeError("The 'name' variable entered was not a string") + + elif not isinstance(favourite_number, int): + raise TypeError("The 'favourite_number' variable entered was not an integer") + + else: + raise TypeError( + "The 'name' and 'favourite_number' variables have incorrect types" + ) diff --git a/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.repo_name.lower().replace(' ', '_').replace('-', '_') }}/run_pipeline.py b/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.repo_name.lower().replace(' ', '_').replace('-', '_') }}/run_pipeline.py new file mode 100644 index 0000000..e33bb35 --- /dev/null +++ b/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.repo_name.lower().replace(' ', '_').replace('-', '_') }}/run_pipeline.py @@ -0,0 +1,25 @@ +import yaml + +from {{ cookiecutter.repo_name.lower().replace(' ', '_').replace('-', '_') }}.example_modules.example_module import ( + hello_world, + print_string, + print_favourite_number, +) + + +def run_pipeline(): + """This is the main function that runs the pipeline""" + with open("src/{{ cookiecutter.repo_name.lower().replace(' ', '_').replace('-', '_') }}/example_config.yml", "r") as file: + example_config = yaml.safe_load(file) + + name = example_config["user_name"] + company = example_config["company"] + favourite_number = example_config["favourite_number"] + + hello_string = hello_world(name, company) + print_string(hello_string) + print_favourite_number(favourite_number, name) + + +if __name__ == "__main__": + run_pipeline() diff --git a/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.repo_name.lower().replace(' ', '_').replace('-', '_') }}/utils/example_module.py b/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.repo_name.lower().replace(' ', '_').replace('-', '_') }}/utils/example_module.py deleted file mode 100644 index 4f86b44..0000000 --- a/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.repo_name.lower().replace(' ', '_').replace('-', '_') }}/utils/example_module.py +++ /dev/null @@ -1,6 +0,0 @@ -def hello_world(name): - if isinstance(name, str): - string = "hello world and hello " + name - return string - else: - raise TypeError("The 'name' variable entered was not a string") diff --git a/{{ cookiecutter.repo_name }}/tests/test_example_module.py b/{{ cookiecutter.repo_name }}/tests/test_example_module.py index 708f858..eda2195 100644 --- a/{{ cookiecutter.repo_name }}/tests/test_example_module.py +++ b/{{ cookiecutter.repo_name }}/tests/test_example_module.py @@ -1,11 +1,54 @@ -from {{ cookiecutter.repo_name.lower().replace(' ', '_').replace('-', '_') }}.utils.example_module import hello_world import pytest +from {{ cookiecutter.repo_name.lower().replace(' ', '_').replace('-', '_') }}.example_modules.example_module import ( + hello_world, + print_string, + print_favourite_number, +) + class TestHelloWorld: def test_string_concat(self): - assert hello_world("ONS") == "hello world and hello ONS" + assert hello_world("Bob", "ONS") == "Hello Bob and hello everyone at ONS" + + def test_name_type_error(self): + with pytest.raises(TypeError): + hello_world(1, "ONS") + + def test_company_type_error(self): + with pytest.raises(TypeError): + hello_world("Bob", 1) + + def test_name_and_company_type_error(self): + with pytest.raises(TypeError): + hello_world(1, 2) + + +class TestPrintString: + def test_print_statements(self, capsys): + print_string("Hello world and hello ONS") + captured = capsys.readouterr() + assert captured.out == "Hello world and hello ONS\n" def test_type_error(self): with pytest.raises(TypeError): - hello_world(1) + print_string(1) + + +class TestPrintFavouriteNumber: + def test_string_concat(self, capsys): + print_favourite_number(7, "Bob") + captured = capsys.readouterr() + assert captured.out == "Bob's favourite number is 7!\n" + + def test_name_type_error(self): + with pytest.raises(TypeError): + print_favourite_number("7", "Bob") + + def test_company_type_error(self): + with pytest.raises(TypeError): + print_favourite_number(7, 1) + + def test_name_and_company_type_error(self): + with pytest.raises(TypeError): + print_favourite_number("1", 2) From 41f6d058086cf1c9eb6fae62b9d3bf62fa704b64 Mon Sep 17 00:00:00 2001 From: joshlynchONS <118733244+joshlynchONS@users.noreply.github.com> Date: Tue, 28 Mar 2023 15:24:00 +0100 Subject: [PATCH 07/13] docs: update docs in contrubiting guide for running pytests --- CONTRIBUTING.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6e1730c..58b760f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -65,12 +65,11 @@ enforced in all cases, for example with long hyperlinks. ## Testing [Tests are written using the `pytest` framework][pytest], with its configuration in the -`pyproject.toml` file. Note, only tests in the `tests`, and -`{{ cookiecutter.repo_name }}/tests` folders folder are run. To run the tests, enter +`pyproject.toml` file. Note, only the `tests` folder in the root direcrtory of this project are to run. To run the tests, enter the following command in your terminal: ```shell -pytest +pytest tests ``` ### Code coverage From 1b0652bd5fdd539aec785364b510287f2e605f3c Mon Sep 17 00:00:00 2001 From: joshlynchONS <118733244+joshlynchONS@users.noreply.github.com> Date: Thu, 30 Mar 2023 10:52:08 +0100 Subject: [PATCH 08/13] Fix: Fix isort skipping * fix: fix isort skipping * chore: update code style of imports * chore: fix import style in test folder * fix: add exclude to pre commit for example test * fix: fix import order to match isort * fix: exclude run_pipeline from initial pre-commits --- .pre-commit-config.yaml | 2 +- tests/test_documentation_build.py | 1 + tests/test_env.py | 3 ++- tests/test_govcookiecutter_creation.py | 5 +++-- tests/test_govcookiecutter_injected_variables.py | 2 +- tests/test_post_gen_project/conftest.py | 1 + .../test_delete_files_and_folders.py | 4 +++- .../test_post_gen_project/test_parse_features_json.py | 6 ++++-- tests/test_post_gen_project/test_set_aqa_framework.py | 4 +++- .../test_set_request_template.py | 3 ++- .../test_check_valid_email_address_format.py | 3 ++- {{ cookiecutter.repo_name }}/.pre-commit-config.yaml | 11 +++++++++-- .../run_pipeline.py | 2 +- .../tests/test_example_module.py | 2 +- 14 files changed, 34 insertions(+), 15 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fdb68b6..3fd040e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -16,8 +16,8 @@ repos: hooks: - id: isort name: isort - Sort Python imports (auto-fixes) - types: [ cython, pyi, python ] args: [ "--profile", "black", "--filter-files" ] + exclude: test_example_module.py|run_pipeline.py - repo: https://github.com/psf/black rev: 22.8.0 # Replace by any tag/version: https://github.com/psf/black/tags hooks: diff --git a/tests/test_documentation_build.py b/tests/test_documentation_build.py index b511766..34b821a 100644 --- a/tests/test_documentation_build.py +++ b/tests/test_documentation_build.py @@ -1,4 +1,5 @@ from pathlib import Path + from sphinx.cmd.build import main diff --git a/tests/test_env.py b/tests/test_env.py index cb0310b..6d8214b 100644 --- a/tests/test_env.py +++ b/tests/test_env.py @@ -1,7 +1,8 @@ -from dotenv import dotenv_values from pathlib import Path from typing import Dict, List +from dotenv import dotenv_values + # Define a path to the `govcookiecutter` template directory, and its `.env` file DIR_TEMPLATE = Path("{{ cookiecutter.repo_name }}") PATH_TEMPLATE_ENV = DIR_TEMPLATE.joinpath(".env") diff --git a/tests/test_govcookiecutter_creation.py b/tests/test_govcookiecutter_creation.py index 02dc21e..63afaf3 100644 --- a/tests/test_govcookiecutter_creation.py +++ b/tests/test_govcookiecutter_creation.py @@ -1,7 +1,8 @@ -from sphinx.cmd.build import main +import re from typing import Dict + import pytest -import re +from sphinx.cmd.build import main @pytest.mark.parametrize("test_input_repository_hosting_platform", ["GitHub", "GitLab"]) diff --git a/tests/test_govcookiecutter_injected_variables.py b/tests/test_govcookiecutter_injected_variables.py index 578db64..d2f5f29 100644 --- a/tests/test_govcookiecutter_injected_variables.py +++ b/tests/test_govcookiecutter_injected_variables.py @@ -1,8 +1,8 @@ from collections import Counter from pathlib import Path from typing import Dict, List, Union -import pytest +import pytest # Define the expected counts for each cookiecutter variable - any counts that deviate # because of other variables are listed at the end of each dictionary diff --git a/tests/test_post_gen_project/conftest.py b/tests/test_post_gen_project/conftest.py index d6a53f4..0f6bd23 100644 --- a/tests/test_post_gen_project/conftest.py +++ b/tests/test_post_gen_project/conftest.py @@ -1,5 +1,6 @@ from pathlib import Path from typing import List + import pytest diff --git a/tests/test_post_gen_project/test_delete_files_and_folders.py b/tests/test_post_gen_project/test_delete_files_and_folders.py index 1648262..a4b40e3 100644 --- a/tests/test_post_gen_project/test_delete_files_and_folders.py +++ b/tests/test_post_gen_project/test_delete_files_and_folders.py @@ -1,9 +1,11 @@ -from hooks.post_gen_project import delete_files_and_folders from pathlib import Path from typing import List from unittest.mock import MagicMock, call + import pytest +from hooks.post_gen_project import delete_files_and_folders + @pytest.fixture def patch_rmtree(mocker) -> MagicMock: diff --git a/tests/test_post_gen_project/test_parse_features_json.py b/tests/test_post_gen_project/test_parse_features_json.py index a8e59ca..fe428b0 100644 --- a/tests/test_post_gen_project/test_parse_features_json.py +++ b/tests/test_post_gen_project/test_parse_features_json.py @@ -1,10 +1,12 @@ +import json from collections import Counter -from hooks.post_gen_project import parse_features_json from pathlib import Path from typing import Dict, List -import json + import pytest +from hooks.post_gen_project import parse_features_json + # Keys for the JSON file JSON_BASE_KEYS = ["name", "remove", "resources"] diff --git a/tests/test_post_gen_project/test_set_aqa_framework.py b/tests/test_post_gen_project/test_set_aqa_framework.py index 327bb33..74b0d4a 100644 --- a/tests/test_post_gen_project/test_set_aqa_framework.py +++ b/tests/test_post_gen_project/test_set_aqa_framework.py @@ -1,8 +1,10 @@ -from hooks.post_gen_project import delete_files_and_folders, set_aqa_framework from pathlib import Path from unittest.mock import MagicMock + import pytest +from hooks.post_gen_project import delete_files_and_folders, set_aqa_framework + @pytest.fixture def patch_delete_files_and_folders(mocker) -> MagicMock: diff --git a/tests/test_post_gen_project/test_set_request_template.py b/tests/test_post_gen_project/test_set_request_template.py index 1d53019..fd0fb6f 100644 --- a/tests/test_post_gen_project/test_set_request_template.py +++ b/tests/test_post_gen_project/test_set_request_template.py @@ -1,7 +1,8 @@ -from hooks.post_gen_project import set_request_template from pathlib import Path + import pytest +from hooks.post_gen_project import set_request_template # Define test cases for the `TestSetRequestTemplate` test class args_test_set_request_template_valueerror_for_bad_repository_hosting_platform = [ diff --git a/tests/test_pre_gen_project/test_check_valid_email_address_format.py b/tests/test_pre_gen_project/test_check_valid_email_address_format.py index 11bdd93..ffb765a 100644 --- a/tests/test_pre_gen_project/test_check_valid_email_address_format.py +++ b/tests/test_pre_gen_project/test_check_valid_email_address_format.py @@ -1,6 +1,7 @@ -from hooks.pre_gen_project import check_valid_email_address_format import pytest +from hooks.pre_gen_project import check_valid_email_address_format + # Define test cases for the `TestCheckValidEmailAddressFormat` test class args_invalid_email_addresses = ["hello.world", "foo_bar"] args_valid_email_addresses = ["hello@world.com", "foo@bar"] diff --git a/{{ cookiecutter.repo_name }}/.pre-commit-config.yaml b/{{ cookiecutter.repo_name }}/.pre-commit-config.yaml index 63fc530..99ef76b 100644 --- a/{{ cookiecutter.repo_name }}/.pre-commit-config.yaml +++ b/{{ cookiecutter.repo_name }}/.pre-commit-config.yaml @@ -24,8 +24,15 @@ repos: rev: 5.12.0 hooks: - id: isort - name: isort - Sort Python imports (auto-fixes) - types: [ cython, pyi, python ] + name: isort - Sort python imports (auto-fixes) + args: [ "--profile", "black", "--filter-files" ] + - id: isort + name: isort - Sort cython imports (auto-fixes) + types: [cython] + args: [ "--profile", "black", "--filter-files" ] + - id: isort + name: isort - Sort pyi imports (auto-fixes) + types: [pyi] args: [ "--profile", "black", "--filter-files" ] - repo: https://github.com/psf/black rev: 22.8.0 # Replace by any tag/version: https://github.com/psf/black/tags diff --git a/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.repo_name.lower().replace(' ', '_').replace('-', '_') }}/run_pipeline.py b/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.repo_name.lower().replace(' ', '_').replace('-', '_') }}/run_pipeline.py index e33bb35..9d4a367 100644 --- a/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.repo_name.lower().replace(' ', '_').replace('-', '_') }}/run_pipeline.py +++ b/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.repo_name.lower().replace(' ', '_').replace('-', '_') }}/run_pipeline.py @@ -2,8 +2,8 @@ from {{ cookiecutter.repo_name.lower().replace(' ', '_').replace('-', '_') }}.example_modules.example_module import ( hello_world, - print_string, print_favourite_number, + print_string, ) diff --git a/{{ cookiecutter.repo_name }}/tests/test_example_module.py b/{{ cookiecutter.repo_name }}/tests/test_example_module.py index eda2195..a480ae6 100644 --- a/{{ cookiecutter.repo_name }}/tests/test_example_module.py +++ b/{{ cookiecutter.repo_name }}/tests/test_example_module.py @@ -2,8 +2,8 @@ from {{ cookiecutter.repo_name.lower().replace(' ', '_').replace('-', '_') }}.example_modules.example_module import ( hello_world, - print_string, print_favourite_number, + print_string, ) From df58ec82b1f18f8aa671f3e2c2f4786e30836406 Mon Sep 17 00:00:00 2001 From: joshlynchONS <118733244+joshlynchONS@users.noreply.github.com> Date: Mon, 24 Apr 2023 18:35:04 +0100 Subject: [PATCH 09/13] add: add version to setup.cfg file --- tests/test_govcookiecutter_injected_variables.py | 4 ++-- {{ cookiecutter.repo_name }}/setup.cfg | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test_govcookiecutter_injected_variables.py b/tests/test_govcookiecutter_injected_variables.py index d2f5f29..6640bd9 100644 --- a/tests/test_govcookiecutter_injected_variables.py +++ b/tests/test_govcookiecutter_injected_variables.py @@ -42,7 +42,7 @@ } PROJECT_VERSION_COUNT = { '"{{ cookiecutter.project_version }}"': 2, - "{{ cookiecutter.project_version }}": 0, + "{{ cookiecutter.project_version }}": 1, } USING_R_NO_COUNT = { "https://github.com/lorenzwalthert/precommit": 0, @@ -179,7 +179,7 @@ def recursive_open_and_count_search_terms( ( "project_version", "project_version_2", - {**PROJECT_VERSION_COUNT, "{{ cookiecutter.project_version }}": 1}, + {**PROJECT_VERSION_COUNT, "{{ cookiecutter.project_version }}": 2}, {"using_R": "Yes"}, ), ("using_R", "No", USING_R_NO_COUNT, {}), diff --git a/{{ cookiecutter.repo_name }}/setup.cfg b/{{ cookiecutter.repo_name }}/setup.cfg index f64726c..57d7892 100644 --- a/{{ cookiecutter.repo_name }}/setup.cfg +++ b/{{ cookiecutter.repo_name }}/setup.cfg @@ -1,6 +1,7 @@ [metadata] name = {{ cookiecutter.repo_name }} description = a brief description +version = {{ cookiecutter.project_version }} author = {{ cookiecutter.organisation_handle }} platforms = win32 classifiers = From ce8e083cc99c957e135f85726cf5bc7e1394ddc1 Mon Sep 17 00:00:00 2001 From: joshlynchONS <118733244+joshlynchONS@users.noreply.github.com> Date: Mon, 12 Jun 2023 14:55:36 +0100 Subject: [PATCH 10/13] add: Add dev dependencies to user setup.cfg (#88) * add: add dev requirements to the setup.cfg * remove: remove requirements file * docs: update docs to refelect the updated dev requirements * fix: replace requirements install in workflow for pip install * fix: update makefile to use pip install over requiremnets.txt * fix: update makefile to install package as dev or user * fix: remove requirements from makefile and update windows makefile * docs: update docs to use make command and remove requirements.txt file * style: fixed layout issue in readme * style: fix the layout of the initial readme * docs: updated contributing guides inside package --- .../govcookiecutter-template-build.yml | 2 +- README.md | 9 +++++++-- {{ cookiecutter.repo_name }}/Makefile | 20 ++++++++++++------- {{ cookiecutter.repo_name }}/README.md | 16 ++++++++++++--- .../docs/contributor_guide/CONTRIBUTING.md | 11 +++++----- {{ cookiecutter.repo_name }}/make.bat | 20 ++++++++++++------- {{ cookiecutter.repo_name }}/requirements.txt | 8 -------- {{ cookiecutter.repo_name }}/setup.cfg | 12 +++++++---- 8 files changed, 60 insertions(+), 38 deletions(-) delete mode 100644 {{ cookiecutter.repo_name }}/requirements.txt diff --git a/.github/workflows/govcookiecutter-template-build.yml b/.github/workflows/govcookiecutter-template-build.yml index 1668c38..964bd53 100644 --- a/.github/workflows/govcookiecutter-template-build.yml +++ b/.github/workflows/govcookiecutter-template-build.yml @@ -69,7 +69,7 @@ jobs: make docs elif [ "$RUNNER_OS" == "Windows" ]; then pip install -U pip setuptools - pip install -r requirements.txt + pip install -e .[dev] pre-commit install sphinx-build -b html ./docs ./docs/_build else diff --git a/README.md b/README.md index bb61781..3e78761 100644 --- a/README.md +++ b/README.md @@ -40,19 +40,24 @@ Once you've answered all the prompts, your project will be created. Then: 1. Set up a Python virtual environment — [there are many ways to set up a virtual environment][pluralsight], so we'll let you decide what's best for you! + 2. In your terminal, navigate to your new project, and initialise Git ```shell git init ``` + 3. Install the necessary packages using `pip` and the pre-commit hooks: ```shell - pip install -r requirements.txt + python -m pip install -U pip setuptools + python -m pip install -e .[dev] pre-commit install ``` + or use the `make` command: ```shell - make requirements + make install_dev ``` + 4. Stage all your project files, and make your first commit ```shell git add . diff --git a/{{ cookiecutter.repo_name }}/Makefile b/{{ cookiecutter.repo_name }}/Makefile index d22da3f..24203d8 100644 --- a/{{ cookiecutter.repo_name }}/Makefile +++ b/{{ cookiecutter.repo_name }}/Makefile @@ -5,32 +5,38 @@ docs docs_check_external_links help + install + install_dev prepare_docs_folder - requirements .DEFAULT_GOAL := help -## Install the Python requirements for contributors, and install pre-commit hooks -requirements: +## Install the Python package for contributors, and install pre-commit hooks +install_dev: python -m pip install -U pip setuptools - python -m pip install -r requirements.txt + python -m pip install -e .[dev] pre-commit install +## Install the Python package for users +install: + python -m pip install -U pip setuptools + python -m pip install -e . + ## Create a `docs/_build` folder, if it does not exist. Otherwise delete any sub-folders and their contents within it prepare_docs_folder: if [ ! -d "./docs/_build" ]; then mkdir ./docs/_build; fi find ./docs/_build -mindepth 1 -maxdepth 1 -type d -exec rm -rf {} \; ## Compile the Sphinx documentation in HTML format in the docs/_build folder from a clean build -docs: prepare_docs_folder requirements +docs: prepare_docs_folder install_dev sphinx-build -b html ./docs ./docs/_build ## Check external links in the Sphinx documentation using linkcheck in the docs/_build folder from a clean build -docs_check_external_links: prepare_docs_folder requirements +docs_check_external_links: prepare_docs_folder install_dev sphinx-build -b linkcheck ./docs ./docs/_build ## Run code coverage -coverage: requirements +coverage: install_dev coverage run -m pytest ## Run code coverage, and produce a HTML output diff --git a/{{ cookiecutter.repo_name }}/README.md b/{{ cookiecutter.repo_name }}/README.md index e87a172..301c4e0 100644 --- a/{{ cookiecutter.repo_name }}/README.md +++ b/{{ cookiecutter.repo_name }}/README.md @@ -20,8 +20,13 @@ Whilst in the root folder, in the command prompt, you can install the package an using: ```shell +python -m pip install -U pip setuptools pip install -e . ``` +or use the `make` command: +```shell +make install +``` This installs an editable version of the package. Meaning, when you update the package code, you do not have to reinstall it for the changes to take effect. @@ -79,10 +84,15 @@ contributing guidelines][contributing]. credentials](#required-secrets-and-credentials) - [load environment variables][docs-loading-environment-variables] from `.env` -To install the Python requirements, open your terminal and enter: - +To install the contributing requirements, open your terminal and enter: +```shell +python -m pip install -U pip setuptools +pip install -e .[dev] +pre-commit install +``` +or use the `make` command: ```shell -pip install -r requirements.txt +make install_dev ``` ## Acknowledgements diff --git a/{{ cookiecutter.repo_name }}/docs/contributor_guide/CONTRIBUTING.md b/{{ cookiecutter.repo_name }}/docs/contributor_guide/CONTRIBUTING.md index 621f328..a9d09bd 100644 --- a/{{ cookiecutter.repo_name }}/docs/contributor_guide/CONTRIBUTING.md +++ b/{{ cookiecutter.repo_name }}/docs/contributor_guide/CONTRIBUTING.md @@ -10,18 +10,17 @@ we'd be happy to help! ## Getting started -To start contributing, open your terminal, and install the required Python packages, -and [pre-commit hooks][pre-commit] using: +To start contributing, open your terminal and install the package and +[pre-commit hooks][pre-commit] using: ```shell -pip install -r requirements.txt +pip install -e .[dev] pre-commit install ``` -or the `make` command: - +or use the `make` command: ```shell -make requirements +make install_dev ``` The pre-commit hooks are a security feature to ensure, for example, no secrets[^1], diff --git a/{{ cookiecutter.repo_name }}/make.bat b/{{ cookiecutter.repo_name }}/make.bat index 30ab66b..6f79204 100644 --- a/{{ cookiecutter.repo_name }}/make.bat +++ b/{{ cookiecutter.repo_name }}/make.bat @@ -2,7 +2,8 @@ IF /I "%1"=="" GOTO .DEFAULT_GOAL -IF /I "%1"=="requirements" GOTO requirements +IF /I "%1"=="install" GOTO install +IF /I "%1"=="install_dev" GOTO install_dev IF /I "%1"=="prepare_docs_folder" GOTO prepare_docs_folder IF /I "%1"=="docs" GOTO docs IF /I "%1"=="docs_check_external_links" GOTO docs_check_external_links @@ -15,12 +16,17 @@ GOTO error :.DEFAULT_GOAL GOTO help -:requirements +:install_dev python -m pip install -U pip setuptools - python -m pip install -r requirements.txt + python -m pip install -e .[dev] pre-commit install GOTO :EOF +:install + python -m pip install -U pip setuptools + python -m pip install -e . + GOTO :EOF + :prepare_docs_folder IF exist "./docs/_build" ( rmdir /s /q "./docs/_build/" ) mkdir ".\docs\_build" @@ -28,18 +34,18 @@ GOTO error :docs CALL make.bat prepare_docs_folder - CALL make.bat requirements + CALL make.bat install_dev sphinx-build -b html ./docs ./docs/_build GOTO :EOF :docs_check_external_links CALL make.bat prepare_docs_folder - CALL make.bat requirements + CALL make.bat install_dev sphinx-build -b linkcheck ./docs ./docs/_build GOTO :EOF :coverage - CALL make.bat requirements + CALL make.bat install_dev coverage run -m pytest GOTO :EOF @@ -54,7 +60,7 @@ GOTO error GOTO :EOF :help - ECHO make: Use one of the following commands: requirements, docs, docs_check_external_links, coverage, coverage_html, coverage_xml. + ECHO make: Use one of the following commands: install, install_dev, docs, docs_check_external_links, coverage, coverage_html, coverage_xml. GOTO :EOF :error diff --git a/{{ cookiecutter.repo_name }}/requirements.txt b/{{ cookiecutter.repo_name }}/requirements.txt deleted file mode 100644 index 7326347..0000000 --- a/{{ cookiecutter.repo_name }}/requirements.txt +++ /dev/null @@ -1,8 +0,0 @@ -coverage -detect-secrets==1.0.3 -myst-parser -pre-commit -pytest -python-dotenv -Sphinx -toml diff --git a/{{ cookiecutter.repo_name }}/setup.cfg b/{{ cookiecutter.repo_name }}/setup.cfg index 57d7892..2c37313 100644 --- a/{{ cookiecutter.repo_name }}/setup.cfg +++ b/{{ cookiecutter.repo_name }}/setup.cfg @@ -14,6 +14,14 @@ classifiers = packages = {{ cookiecutter.repo_name.lower().replace(' ', '_').replace('-', '_') }} install_requires = + pyyaml +python_requires = >=3.6 +package_dir = + =./src +zip_safe = no + +[options.extras_require] +dev = coverage detect-secrets == 1.0.3 myst-parser @@ -23,7 +31,3 @@ install_requires = python-dotenv Sphinx toml -python_requires = >=3.6 -package_dir = - =./src -zip_safe = no From d57f84f6e3d5dd7f926fd0b5390701065be65c3e Mon Sep 17 00:00:00 2001 From: Jacob Cole <77634686+Jacobb164@users.noreply.github.com> Date: Wed, 21 Jun 2023 14:59:17 +0100 Subject: [PATCH 11/13] update setup.cfg description section to match input settings (#91) * update setup.cfg description section to match input settings * Add python -m to README instructions * Fix broken links in contributing.md --- CONTRIBUTING.md | 7 ++----- README.md | 4 ++-- tests/test_govcookiecutter_injected_variables.py | 4 ++-- .../docs/contributor_guide/CONTRIBUTING.md | 7 ++----- {{ cookiecutter.repo_name }}/setup.cfg | 2 +- 5 files changed, 9 insertions(+), 15 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 58b760f..da99355 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -37,10 +37,7 @@ on pre-commit hooks see our documentation][docs-pre-commit-hooks]. ### Git and GitHub -We use Git to version control the source code. [Please read the GDS Way for details on -Git best practice][gds-way-git]. This includes how to write good commit messages, use -`git rebase` for local branches and `git merge --no-ff` for merges, as well as using -`git push --force-with-lease` instead of `git push -f`. +We use Git to version control the source code. [Please read the Quality assurance of code for analysis and research for details on Git best practice][duck-book-version-control]. This includes how to write good commit messages, how to branch correctly and solving merge conflicts. [If you want to modify the `.gitignore` files, see the template documentation][docs-updating-gitignore] for further details. @@ -123,7 +120,7 @@ the `README.md` file in that folder. [docs-write-sphinx-documentation]: https://github.com/best-practice-and-impact/govcookiecutter/blob/main/%7B%7B%20cookiecutter.repo_name%20%7D%7D/docs/contributor_guide/writing_sphinx_documentation.md [email]: mailto:gsshelp@statistics.gov.uk [gds-way]: https://gds-way.cloudapps.digital/ -[gds-way-git]: https://gds-way.cloudapps.digital/standards/source-code.html +[gds-way-git]: https://www.gov.uk/service-manual/technology/maintaining-version-control-in-coding [gds-way-python]: https://gds-way.cloudapps.digital/manuals/programming-languages/python/python.html#python-style-guide [govcookiecutter]: https://github.com/best-practice-and-impact/govcookiecutter [myst]: https://myst-parser.readthedocs.io/ diff --git a/README.md b/README.md index 3e78761..c17d181 100644 --- a/README.md +++ b/README.md @@ -23,13 +23,13 @@ terminal, navigate to the directory where you want your new repository to exist. run the following command for the latest stable release: ```shell -cookiecutter https://github.com/best-practice-and-impact/govcookiecutter.git +python -m cookiecutter https://github.com/best-practice-and-impact/govcookiecutter.git ``` or for a specific branch, tag, or commit SHA `{SPECIFIC}`, run: ```shell -cookiecutter https://github.com/best-practice-and-impact/govcookiecutter.git --checkout {SPECIFIC} +python -m cookiecutter https://github.com/best-practice-and-impact/govcookiecutter.git --checkout {SPECIFIC} ``` Follow the prompts; if you are asked to re-download `govcookiecutter`, input `yes`. diff --git a/tests/test_govcookiecutter_injected_variables.py b/tests/test_govcookiecutter_injected_variables.py index 6640bd9..a919a5c 100644 --- a/tests/test_govcookiecutter_injected_variables.py +++ b/tests/test_govcookiecutter_injected_variables.py @@ -38,7 +38,7 @@ } OVERVIEW_COUNT = { '"{{ cookiecutter.overview }}",': 0, - "{{ cookiecutter.overview }}": 1, + "{{ cookiecutter.overview }}": 2, } PROJECT_VERSION_COUNT = { '"{{ cookiecutter.project_version }}"': 2, @@ -172,7 +172,7 @@ def recursive_open_and_count_search_terms( ( "overview", "overview_1", - {**OVERVIEW_COUNT, "{{ cookiecutter.overview }}": 2}, + {**OVERVIEW_COUNT, "{{ cookiecutter.overview }}": 3}, {"using_R": "Yes"}, ), ("project_version", "project_version_1", PROJECT_VERSION_COUNT, {"using_R": "No"}), diff --git a/{{ cookiecutter.repo_name }}/docs/contributor_guide/CONTRIBUTING.md b/{{ cookiecutter.repo_name }}/docs/contributor_guide/CONTRIBUTING.md index a9d09bd..689d89a 100644 --- a/{{ cookiecutter.repo_name }}/docs/contributor_guide/CONTRIBUTING.md +++ b/{{ cookiecutter.repo_name }}/docs/contributor_guide/CONTRIBUTING.md @@ -37,10 +37,7 @@ documentation][docs-pre-commit-hooks]. ### Git and GitHub -We use Git to version control the source code. [Please read the GDS Way for details on -Git best practice][gds-way-git]. This includes how to write good commit messages, use -`git rebase` for local branches and `git merge --no-ff` for merges, as well as using -`git push --force-with-lease` instead of `git push -f`. +We use Git to version control the source code. [Please read the Quality assurance of code for analysis and research for details on Git best practice][duck-book-version-control]. This includes how to write good commit messages, how to branch correctly and solving merge conflicts. [If you want to modify the `.gitignore` files, see the template documentation][docs-updating-gitignore] for further details. @@ -115,7 +112,7 @@ build the documentation into an accessible, searchable website. [docs-write-accessible-documentation]: https://github.com/best-practice-and-impact/govcookiecutter/blob/main/%7B%7B%20cookiecutter.repo_name%20%7D%7D/docs/contributor_guide/writing_accessible_documentation.md [docs-write-sphinx-documentation]: https://github.com/best-practice-and-impact/govcookiecutter/blob/main/%7B%7B%20cookiecutter.repo_name%20%7D%7D/docs/contributor_guide/writing_sphinx_documentation.md [gds-way]: https://gds-way.cloudapps.digital/ -[gds-way-git]: https://gds-way.cloudapps.digital/standards/source-code.html +[duck-book-version-control]: https://best-practice-and-impact.github.io/qa-of-code-guidance/version_control.html [gds-way-python]: https://gds-way.cloudapps.digital/manuals/programming-languages/python/python.html#python-style-guide [myst]: https://myst-parser.readthedocs.io/ [pre-commit]: https://pre-commit.com diff --git a/{{ cookiecutter.repo_name }}/setup.cfg b/{{ cookiecutter.repo_name }}/setup.cfg index 2c37313..7c79175 100644 --- a/{{ cookiecutter.repo_name }}/setup.cfg +++ b/{{ cookiecutter.repo_name }}/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = {{ cookiecutter.repo_name }} -description = a brief description +description = {{ cookiecutter.overview }} version = {{ cookiecutter.project_version }} author = {{ cookiecutter.organisation_handle }} platforms = win32 From e47aad3d5d07c47aae521e42506b25e16514743e Mon Sep 17 00:00:00 2001 From: Jacob Cole <77634686+Jacobb164@users.noreply.github.com> Date: Thu, 21 Sep 2023 11:55:24 +0100 Subject: [PATCH 12/13] Add bandit to pre-commit (#94) * Added bandit to pre-commit hooks * Testing requirements * Testing building * Restores bandit * Add blank line * Adds jinja2-time to requirements * Adds missing dash * Adds jinja2-time to template as well as higher level * Fixes typo * Testing * Testing with jinja2_time * Testing with jinja2-time * Testing with both * Adds jinja2_time to package * Makes jinja2_time a dev package * Testing * Testing * Removing extensions * Tidies up a few things --------- Co-authored-by: Milne --- cookiecutter.json | 2 -- requirements.txt | 2 ++ {{ cookiecutter.repo_name }}/.pre-commit-config.yaml | 7 +++++++ {{ cookiecutter.repo_name }}/pyproject.toml | 5 +++++ 4 files changed, 14 insertions(+), 2 deletions(-) diff --git a/cookiecutter.json b/cookiecutter.json index af0116e..cd84be1 100644 --- a/cookiecutter.json +++ b/cookiecutter.json @@ -1,6 +1,4 @@ { - "_extensions": ["jinja2_time.TimeExtension"], - "organisation_name": "Your public sector organisation name, for example Government Digital Service", "repository_hosting_platform": ["GitHub", "GitLab"], "organisation_handle": "Your GitHub/GitLab organisation name, for example ukgovdatascience", diff --git a/requirements.txt b/requirements.txt index db5b334..5539863 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,11 @@ +bandit cookiecutter coverage detect-secrets==1.0.3 #pytest-cookies git+https://github.com/Jacobb164/pytest-cookies govuk-tech-docs-sphinx-theme +jinja2-time myst-parser pre-commit pytest diff --git a/{{ cookiecutter.repo_name }}/.pre-commit-config.yaml b/{{ cookiecutter.repo_name }}/.pre-commit-config.yaml index 99ef76b..b8d0ac1 100644 --- a/{{ cookiecutter.repo_name }}/.pre-commit-config.yaml +++ b/{{ cookiecutter.repo_name }}/.pre-commit-config.yaml @@ -67,6 +67,13 @@ repos: name: detect-secrets - Detect secrets in staged code args: [ "--baseline", ".secrets.baseline" ] exclude: .*/tests/.*|^\.cruft\.json$ + - repo: https://github.com/PyCQA/bandit + rev: '1.7.5' + hooks: + - id: bandit + name: bandit - Checks for vulnerabilities + args: ["-c", "pyproject.toml"] + additional_dependencies: ["bandit[toml]"] {% if cookiecutter.using_R == "Yes" %} # R specific hooks: https://github.com/lorenzwalthert/precommit - repo: https://github.com/lorenzwalthert/precommit diff --git a/{{ cookiecutter.repo_name }}/pyproject.toml b/{{ cookiecutter.repo_name }}/pyproject.toml index 822946b..64b919e 100644 --- a/{{ cookiecutter.repo_name }}/pyproject.toml +++ b/{{ cookiecutter.repo_name }}/pyproject.toml @@ -23,3 +23,8 @@ doctest_optionflags = "NORMALIZE_WHITESPACE" testpaths = [ "./tests" ] + +# `bandit' configurations +[tool.bandit] +exclude_dirs = ["tests", "docs"] +skips = [] From 92971f473633f016ae3a37e7e592e97aba63adf4 Mon Sep 17 00:00:00 2001 From: Jordan Day Date: Wed, 4 Dec 2024 09:23:47 +0000 Subject: [PATCH 13/13] Updating gitignore to include csv and excel filetypes --- .gitignore | 6 ++++++ {{ cookiecutter.repo_name }}/.gitignore | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/.gitignore b/.gitignore index 2efb05e..f001c3b 100644 --- a/.gitignore +++ b/.gitignore @@ -864,3 +864,9 @@ example/* # Ignore Sphinx documentation link checking folder docs/_linkcheck/ + +# Commonly used data file extensions + +*.csv +*.xls +*.xlsx diff --git a/{{ cookiecutter.repo_name }}/.gitignore b/{{ cookiecutter.repo_name }}/.gitignore index c9fda26..026051c 100644 --- a/{{ cookiecutter.repo_name }}/.gitignore +++ b/{{ cookiecutter.repo_name }}/.gitignore @@ -883,3 +883,9 @@ docs/reference/api/* # Ignore Sphinx documentation link checking folder docs/_linkcheck/ + +# Commonly used data file extensions + +*.csv +*.xls +*.xlsx