From f56376b2d914c1f30bcab713daa55efcf0170271 Mon Sep 17 00:00:00 2001 From: barneydobson Date: Thu, 11 Jan 2024 10:55:01 +0000 Subject: [PATCH] Initial commit --- .github/PULL_REQUEST_TEMPLATE.md | 26 ++++ .github/dependabot.yml | 15 +++ .github/workflows/auto-merge.yml | 21 ++++ .github/workflows/check-links.yml | 15 +++ .github/workflows/ci.yml | 28 +++++ .github/workflows/pre-commit_autoupdate.yml | 20 +++ .github/workflows/publish.yml | 27 ++++ .gitignore | 129 ++++++++++++++++++++ .pre-commit-config.yaml | 20 +++ .vscode/extensions.json | 8 ++ .vscode/settings.json | 12 ++ README.md | 86 +++++++++++++ dev-requirements.txt | 74 +++++++++++ myproject/__init__.py | 2 + myproject/__main__.py | 1 + pyproject.toml | 52 ++++++++ requirements.txt | 6 + tests/__init__.py | 6 + tests/test_myproject.py | 7 ++ 19 files changed, 555 insertions(+) create mode 100644 .github/PULL_REQUEST_TEMPLATE.md create mode 100644 .github/dependabot.yml create mode 100644 .github/workflows/auto-merge.yml create mode 100644 .github/workflows/check-links.yml create mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/pre-commit_autoupdate.yml create mode 100644 .github/workflows/publish.yml create mode 100644 .gitignore create mode 100644 .pre-commit-config.yaml create mode 100644 .vscode/extensions.json create mode 100644 .vscode/settings.json create mode 100644 README.md create mode 100644 dev-requirements.txt create mode 100644 myproject/__init__.py create mode 100644 myproject/__main__.py create mode 100644 pyproject.toml create mode 100644 requirements.txt create mode 100644 tests/__init__.py create mode 100644 tests/test_myproject.py diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 00000000..97fea456 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,26 @@ +# Description + +*Please include a summary of the change and which issue is fixed (if any). Please also +include relevant motivation and context. List any dependencies that are required for +this change.* + +Fixes # (issue) + +## Type of change + +- [ ] Documentation (non-breaking change that adds or improves the documentation) +- [ ] New feature (non-breaking change which adds functionality) +- [ ] Optimization (non-breaking, back-end change that speeds up the code) +- [ ] Bug fix (non-breaking change which fixes an issue) +- [ ] Breaking change (whatever its nature) + +## Key checklist + +- [ ] All tests pass (eg. `python -m pytest`) +- [ ] The documentation builds and looks OK (eg. `python -m sphinx -b html docs docs/build`) +- [ ] Pre-commit hooks run successfully (eg. `pre-commit run --all-files`) + +## Further checks + +- [ ] Code is commented, particularly in hard-to-understand areas +- [ ] Tests added or an issue has been opened to tackle that in the future. (Indicate issue here: # (issue)) diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..70eb19e6 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,15 @@ +# To get started with Dependabot version updates, you'll need to specify which +# package ecosystems to update and where the package manifests are located. +# Please see the documentation for all configuration options: +# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates + +version: 2 +updates: + - package-ecosystem: "pip" + directory: "/" + schedule: + interval: "monthly" + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "monthly" diff --git a/.github/workflows/auto-merge.yml b/.github/workflows/auto-merge.yml new file mode 100644 index 00000000..9a17d0c9 --- /dev/null +++ b/.github/workflows/auto-merge.yml @@ -0,0 +1,21 @@ +name: Dependabot and Pre-commit auto-merge + +on: + pull_request: + +permissions: + contents: write + pull-requests: write # Needed if in a private repository + +jobs: + auto-merge: + runs-on: ubuntu-latest + if: ${{ github.actor == 'dependabot[bot]' || github.actor == 'pre-commit-ci[bot]' }} + steps: + - name: Enable auto-merge for Dependabot PRs + run: gh pr merge --auto --merge "$PR_URL" + env: + PR_URL: ${{github.event.pull_request.html_url}} + # GitHub provides this variable in the CI env. You don't + # need to add anything to the secrets vault. + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/check-links.yml b/.github/workflows/check-links.yml new file mode 100644 index 00000000..4bfc95de --- /dev/null +++ b/.github/workflows/check-links.yml @@ -0,0 +1,15 @@ +name: Check links in Markdown files +on: + push: + schedule: + - cron: "0 0 * * 1" # midnight every Monday + +jobs: + check-links: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: gaurav-nelson/github-action-markdown-link-check@v1 + with: + use-quiet-mode: 'yes' + use-verbose-mode: 'yes' diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..90abfb4c --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,28 @@ +name: Test + +on: [push, pull_request, workflow_call] + +jobs: + qa: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: pre-commit/action@v3.0.0 + + test: + needs: qa + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ windows-latest, ubuntu-latest, macos-latest ] + python-version: [ "3.10" ] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: pip install -r dev-requirements.txt + - name: Run tests + run: pytest diff --git a/.github/workflows/pre-commit_autoupdate.yml b/.github/workflows/pre-commit_autoupdate.yml new file mode 100644 index 00000000..0688e6c3 --- /dev/null +++ b/.github/workflows/pre-commit_autoupdate.yml @@ -0,0 +1,20 @@ +name: Pre-commit auto-update + +on: + schedule: + - cron: "0 0 * * 1" # midnight every Monday + +jobs: + auto-update: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + - uses: browniebroke/pre-commit-autoupdate-action@main + # - uses: peter-evans/create-pull-request@v5 + # with: + # token: ${{ secrets.GITHUB_TOKEN }} + # branch: update/pre-commit-hooks + # title: Update pre-commit hooks + # commit-message: "chore: update pre-commit hooks" + # body: Update versions of pre-commit hooks to latest version. diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 00000000..2b13faec --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,27 @@ +on: [release] + +jobs: + test: + uses: ./.github/workflows/ci.yml + + # publish: + # runs-on: ubuntu-latest + # needs: test + # # The following steps to build a Docker image and publish to the GitHub container registry on release. Alternatively, can replace with other publising steps (ie. publishing to PyPI, deploying documentation etc.) + # steps: + # - name: Login to GitHub Container Registry + # uses: docker/login-action@v3 + # with: + # registry: ghcr.io + # username: ${{ github.actor }} + # password: ${{ secrets.GITHUB_TOKEN }} + # - name: Get image metadata + # id: meta + # uses: docker/metadata-action@v5 + # with: + # images: ghcr.io/${{ github.repository }} + # - name: Build and push Docker image + # uses: docker/build-push-action@v5 + # with: + # push: true + # tags: ${{ steps.meta.outputs.tags }} diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..b6e47617 --- /dev/null +++ b/.gitignore @@ -0,0 +1,129 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..c12ed7d2 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,20 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.5.0 + hooks: + - id: check-merge-conflict + - id: debug-statements + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: "v0.1.11" + hooks: + - id: ruff + args: [--fix, --exit-non-zero-on-fix] + - repo: https://github.com/pre-commit/mirrors-mypy + rev: "v1.8.0" + hooks: + - id: mypy + - repo: https://github.com/igorshubovych/markdownlint-cli + rev: v0.38.0 + hooks: + - id: markdownlint + args: ["--disable", "MD013", "--"] diff --git a/.vscode/extensions.json b/.vscode/extensions.json new file mode 100644 index 00000000..73e2b16a --- /dev/null +++ b/.vscode/extensions.json @@ -0,0 +1,8 @@ +{ + "recommendations": [ + "ms-python.black-formatter", + "ms-python.flake8", + "ms-python.mypy-type-checker", + "ms-python.python" + ] +} diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 00000000..41a0d657 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,12 @@ +{ + "[python]": { + "editor.formatOnSave": true, + "editor.defaultFormatter": "ms-python.black-formatter" + }, + "editor.rulers": [88], + + // --no-cov needed so that breakpoints work: https://github.com/microsoft/vscode-python/issues/693 + "python.testing.pytestArgs": ["--no-cov"], + "python.testing.unittestEnabled": false, + "python.testing.pytestEnabled": true, +} \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 00000000..7b898f77 --- /dev/null +++ b/README.md @@ -0,0 +1,86 @@ +# pip-tools Template + +[![Test and build](https://github.com/ImperialCollegeLondon/pip-tools-template/actions/workflows/ci.yml/badge.svg)](https://github.com/ImperialCollegeLondon/pip-tools-template/actions/workflows/ci.yml) + +This is a minimal Python 3.10 application that uses [`pip-tools`] for packaging and dependency management. It also provides [`pre-commit`](https://pre-commit.com/) hooks (for for [ruff](https://pypi.org/project/ruff/) and [`mypy`](https://mypy.readthedocs.io/en/stable/)) and automated tests using [`pytest`](https://pytest.org/) and [GitHub Actions](https://github.com/features/actions). Pre-commit hooks are automatically kept updated with a dedicated GitHub Action, this can be removed and replace with [pre-commit.ci](https://pre-commit.ci) if using an public repo. It was developed by the [Imperial College Research Computing Service](https://www.imperial.ac.uk/admin-services/ict/self-service/research-support/rcs/). + +[`pip-tools`] is chosen as a lightweight dependency manager that adheres to the [latest standards](https://peps.python.org/pep-0621/) using `pyproject.toml`. + +## Usage + +To use this repository as a template for your own application: + +1. Click the green "Use this template" button above +2. Name and create your repository +3. Clone your new repository and make it your working directory +4. Replace instances of `myproject` with your own application name. Edit: + - `pyproject.toml` (also change the list of authors here) + - `tests/test_myproject.py` + - Rename `myproject` directory +5. Create and activate a Virtual Environment: + + ```bash + python -m venv .venv + source .venv/bin/activate # with Powershell on Windows: `.venv\Scripts\Activate.ps1` + ``` + +6. Install development requirements: + + ```bash + pip install -r dev-requirements.txt + ``` + +7. Install the git hooks: + + ```bash + pre-commit install + ``` + +8. Run the main app: + + ```bash + python -m myproject + ``` + +9. Run the tests: + + ```bash + pytest + ``` + +### Updating Dependencies + +To add or remove dependencies: + +1. Edit the `dependencies` variables in the `pyproject.toml` file (aim to keep develpment tools separate from the project requirements). +2. Update the requirements files: + - `pip-compile` for `requirements.txt` - the project requirements. + - `pip-compile --extra dev -o dev-requirements.txt` for `dev-requirements.txt` - the development requirements. +3. Sync the files with your installation (install packages): + - `pip-sync dev-requirements.txt requirements.txt` + +To upgrade pinned versions, use the `--upgrade` flag with `pip-compile`. + +Versions can be restricted from updating within the `pyproject.toml` using standard python package version specifiers, i.e. `"black<23"` or `"pip-tools!=6.12.2"` + +### Customising + +All configuration can be customised to your preferences. The key places to make changes +for this are: + +- The `pyproject.toml` file, where you can edit: + - The build system (change from setuptools to other packaging tools like [Hatch](https://hatch.pypa.io/) or [flit](https://flit.pypa.io/)). + - The python version. + - The project dependencies. Extra optional dependencies can be added by adding another list under `[project.optional-dependencies]` (i.e. `doc = ["mkdocs"]`). + - The `mypy` and `pytest` configurations. +- The `.pre-commit-config.yaml` for pre-commit settings. +- The `.github` directory for all the CI configuration. + - This repo uses `pre-commit.ci` to update pre-commit package versions and automatically merges those PRs with the `auto-merge.yml` workflow. + - Note that `pre-commit.ci` is an external service and free for open source repos. For private repos uncomment the commented portion of the `pre-commit_autoupdate.yml` workflow. + +[`pip-tools`]: https://pip-tools.readthedocs.io/en/latest/ + +### Publishing + +The GitHub workflow includes an action to publish on release. +To run this action, uncomment the commented portion of `publish.yml`, and modify the steps for the desired behaviour (publishing a Docker image, publishing to PyPI, deploying documentation etc.) diff --git a/dev-requirements.txt b/dev-requirements.txt new file mode 100644 index 00000000..97f0e549 --- /dev/null +++ b/dev-requirements.txt @@ -0,0 +1,74 @@ +# +# This file is autogenerated by pip-compile with Python 3.11 +# by the following command: +# +# pip-compile --extra=dev --output-file=dev-requirements.txt +# +attrs==23.1.0 + # via pytest-mypy +build==1.0.3 + # via pip-tools +cfgv==3.4.0 + # via pre-commit +click==8.1.7 + # via pip-tools +coverage[toml]==7.3.2 + # via pytest-cov +distlib==0.3.7 + # via virtualenv +filelock==3.12.4 + # via + # pytest-mypy + # virtualenv +identify==2.5.27 + # via pre-commit +iniconfig==2.0.0 + # via pytest +mypy==1.8.0 + # via + # datahub (pyproject.toml) + # pytest-mypy +mypy-extensions==1.0.0 + # via mypy +nodeenv==1.8.0 + # via pre-commit +packaging==23.2 + # via + # build + # pytest +pip-tools==7.3.0 + # via datahub (pyproject.toml) +platformdirs==3.11.0 + # via virtualenv +pluggy==1.3.0 + # via pytest +pre-commit==3.5.0 + # via datahub (pyproject.toml) +pyproject-hooks==1.0.0 + # via build +pytest==7.4.3 + # via + # datahub (pyproject.toml) + # pytest-cov + # pytest-mock + # pytest-mypy +pytest-cov==4.1.0 + # via datahub (pyproject.toml) +pytest-mock==3.12.0 + # via datahub (pyproject.toml) +pytest-mypy==0.10.3 + # via datahub (pyproject.toml) +pyyaml==6.0.1 + # via pre-commit +ruff==0.1.11 + # via datahub (pyproject.toml) +typing-extensions==4.9.0 + # via mypy +virtualenv==20.24.5 + # via pre-commit +wheel==0.41.3 + # via pip-tools + +# The following packages are considered to be unsafe in a requirements file: +# pip +# setuptools diff --git a/myproject/__init__.py b/myproject/__init__.py new file mode 100644 index 00000000..dde32fd0 --- /dev/null +++ b/myproject/__init__.py @@ -0,0 +1,2 @@ +"""The main module for MyProject.""" +__version__ = "0.1.0" diff --git a/myproject/__main__.py b/myproject/__main__.py new file mode 100644 index 00000000..b9f1c2a8 --- /dev/null +++ b/myproject/__main__.py @@ -0,0 +1 @@ +"""The entry point for the myproject program.""" diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..60698cba --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,52 @@ +[build-system] +requires = ["setuptools", "setuptools-scm"] +build-backend = "setuptools.build_meta" + +[tool.setuptools.packages.find] +exclude = ["htmlcov"] # Exclude the coverage report file from setuptools package finder + +[project] +name = "datahub" +version = "0.0.1" +authors = [ + { name = "Jane Doe", email = "jane_doe@some_email.com" }, + { name = "Imperial College London RSE Team", email = "ict-rse-team@imperial.ac.uk" } +] +requires-python = ">=3.10" +dependencies = [] + +[project.optional-dependencies] +dev = [ + "ruff", + "mypy", + "pip-tools", + "pre-commit", + "pytest", + "pytest-cov", + "pytest-mypy", + "pytest-mock" +] + +[tool.mypy] +disallow_any_explicit = true +disallow_any_generics = true +warn_unreachable = true +warn_unused_ignores = true +disallow_untyped_defs = true +exclude = [".venv/"] + +[[tool.mypy.overrides]] +module = "tests.*" +disallow_untyped_defs = false + +[tool.pytest.ini_options] +addopts = "-v --mypy -p no:warnings --cov=myproject --cov-report=html --doctest-modules --ignore=myproject/__main__.py" + +[tool.ruff] +select = ["D", "E", "F", "I"] # pydocstyle, pycodestyle, Pyflakes, isort + +[tool.ruff.per-file-ignores] +"tests/*" = ["D100", "D104"] + +[tool.ruff.pydocstyle] +convention = "google" diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..001b59cd --- /dev/null +++ b/requirements.txt @@ -0,0 +1,6 @@ +# +# This file is autogenerated by pip-compile with Python 3.10 +# by the following command: +# +# pip-compile +# diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 00000000..76d8514b --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1,6 @@ +"""Unit tests for MyProject.""" + +from logging import getLogger + +# Disable flake8 logger as it can be rather verbose +getLogger("flake8").propagate = False diff --git a/tests/test_myproject.py b/tests/test_myproject.py new file mode 100644 index 00000000..3717c353 --- /dev/null +++ b/tests/test_myproject.py @@ -0,0 +1,7 @@ +"""Tests for the main module.""" +from myproject import __version__ + + +def test_version(): + """Check that the version is acceptable.""" + assert __version__ == "0.1.0"