From 0854c3daaa9a92924e71f2cfba1c88b035351646 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Mon, 2 Oct 2023 16:01:36 +0100 Subject: [PATCH] general: migrate to src/ package structure, update CI configs --- .ci/release | 4 +- .ci/run | 32 ++++++++++++---- .github/workflows/main.yml | 29 ++++++++------ Makefile | 2 +- conftest.py | 38 +++++++++++++++++++ pytest.ini | 3 ++ setup.py | 15 ++++---- {orgparse => src/orgparse}/__init__.py | 2 - {orgparse => src/orgparse}/date.py | 0 {orgparse => src/orgparse}/extra.py | 0 {orgparse => src/orgparse}/inline.py | 0 {orgparse => src/orgparse}/node.py | 0 {orgparse => src/orgparse}/py.typed | 0 {orgparse => src/orgparse}/tests/__init__.py | 0 .../orgparse}/tests/data/00_simple.org | 0 .../orgparse}/tests/data/00_simple.py | 0 .../orgparse}/tests/data/01_attributes.org | 0 .../orgparse}/tests/data/01_attributes.py | 0 .../orgparse}/tests/data/02_tree_struct.org | 0 .../orgparse}/tests/data/02_tree_struct.py | 0 .../tests/data/03_repeated_tasks.org | 0 .../orgparse}/tests/data/03_repeated_tasks.py | 0 .../orgparse}/tests/data/04_logbook.org | 0 .../orgparse}/tests/data/04_logbook.py | 0 .../orgparse}/tests/data/05_tags.org | 0 .../orgparse}/tests/data/05_tags.py | 0 .../orgparse}/tests/data/__init__.py | 0 {orgparse => src/orgparse}/tests/test_data.py | 3 +- {orgparse => src/orgparse}/tests/test_date.py | 0 .../orgparse}/tests/test_hugedata.py | 0 {orgparse => src/orgparse}/tests/test_misc.py | 0 {orgparse => src/orgparse}/tests/test_rich.py | 0 tox.ini | 38 ++++++++++++++----- 33 files changed, 124 insertions(+), 42 deletions(-) create mode 100644 conftest.py rename {orgparse => src/orgparse}/__init__.py (98%) rename {orgparse => src/orgparse}/date.py (100%) rename {orgparse => src/orgparse}/extra.py (100%) rename {orgparse => src/orgparse}/inline.py (100%) rename {orgparse => src/orgparse}/node.py (100%) rename {orgparse => src/orgparse}/py.typed (100%) rename {orgparse => src/orgparse}/tests/__init__.py (100%) rename {orgparse => src/orgparse}/tests/data/00_simple.org (100%) rename {orgparse => src/orgparse}/tests/data/00_simple.py (100%) rename {orgparse => src/orgparse}/tests/data/01_attributes.org (100%) rename {orgparse => src/orgparse}/tests/data/01_attributes.py (100%) rename {orgparse => src/orgparse}/tests/data/02_tree_struct.org (100%) rename {orgparse => src/orgparse}/tests/data/02_tree_struct.py (100%) rename {orgparse => src/orgparse}/tests/data/03_repeated_tasks.org (100%) rename {orgparse => src/orgparse}/tests/data/03_repeated_tasks.py (100%) rename {orgparse => src/orgparse}/tests/data/04_logbook.org (100%) rename {orgparse => src/orgparse}/tests/data/04_logbook.py (100%) rename {orgparse => src/orgparse}/tests/data/05_tags.org (100%) rename {orgparse => src/orgparse}/tests/data/05_tags.py (100%) rename {orgparse => src/orgparse}/tests/data/__init__.py (100%) rename {orgparse => src/orgparse}/tests/test_data.py (96%) rename {orgparse => src/orgparse}/tests/test_date.py (100%) rename {orgparse => src/orgparse}/tests/test_hugedata.py (100%) rename {orgparse => src/orgparse}/tests/test_misc.py (100%) rename {orgparse => src/orgparse}/tests/test_rich.py (100%) diff --git a/.ci/release b/.ci/release index 0ec687f..95e41f4 100755 --- a/.ci/release +++ b/.ci/release @@ -21,7 +21,7 @@ import shutil is_ci = os.environ.get('CI') is not None -def main(): +def main() -> None: import argparse p = argparse.ArgumentParser() p.add_argument('--test', action='store_true', help='use test pypi') @@ -29,7 +29,7 @@ def main(): extra = [] if args.test: - extra.extend(['--repository-url', 'https://test.pypi.org/legacy/']) + extra.extend(['--repository', 'testpypi']) root = Path(__file__).absolute().parent.parent os.chdir(root) # just in case diff --git a/.ci/run b/.ci/run index 45d3e82..b2c184d 100755 --- a/.ci/run +++ b/.ci/run @@ -1,10 +1,11 @@ -#!/bin/bash -eu +#!/bin/bash +set -eu cd "$(dirname "$0")" -cd .. +cd .. # git root if ! command -v sudo; then - # CI or Docker sometimes don't have it, so useful to have a dummy + # CI or Docker sometimes doesn't have it, so useful to have a dummy function sudo { "$@" } @@ -12,13 +13,28 @@ fi if [ -n "${CI-}" ]; then # install OS specific stuff here - if [[ "$OSTYPE" == "darwin"* ]]; then + case "$OSTYPE" in + darwin*) # macos : - else + ;; + cygwin* | msys* | win*) + # windows : - fi + ;; + *) + # must be linux? + : + ;; + esac +fi + + +PY_BIN="python3" +# some systems might have python pointing to python3 +if ! command -v python3 &> /dev/null; then + PY_BIN="python" fi -pip3 install --user tox -tox +"$PY_BIN" -m pip install --user tox +"$PY_BIN" -m tox --parallel --parallel-live "$@" diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index acd5204..89338bd 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -10,11 +10,14 @@ on: pull_request: # needed to trigger on others' PRs # Note that people who fork it need to go to "Actions" tab on their fork and click "I understand my workflows, go ahead and enable them". workflow_dispatch: # needed to trigger workflows manually - # todo cron? + # todo cron? + inputs: + debug_enabled: + type: boolean + description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)' + required: false + default: false -env: - # useful for scripts & sometimes tests to know - CI: true jobs: build: @@ -22,15 +25,17 @@ jobs: fail-fast: false matrix: platform: [ubuntu-latest, macos-latest, windows-latest] - python-version: ['3.7', '3.8', '3.9', '3.10', '3.11'] + python-version: ['3.8', '3.9', '3.10', '3.11', '3.12.0-rc.3'] + # vvv just an example of excluding stuff from matrix + # exclude: [{platform: macos-latest, python-version: '3.6'}] runs-on: ${{ matrix.platform }} steps: # ugh https://github.com/actions/toolkit/blob/main/docs/commands.md#path-manipulation - run: echo "$HOME/.local/bin" >> $GITHUB_PATH - - if: ${{ matrix.platform == 'macos-latest' && matrix.python-version == '3.11' }} + # hmm somehow only seems necessary for 3.11 on osx?? run: echo "$HOME/Library/Python/${{ matrix.python-version }}/bin" >> $GITHUB_PATH - uses: actions/setup-python@v4 @@ -41,12 +46,14 @@ jobs: with: submodules: recursive - # uncomment for SSH debugging - # - uses: mxschmitt/action-tmate@v3 + - uses: mxschmitt/action-tmate@v3 + if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }} - - run: .ci/run + # explicit bash command is necessary for Windows CI runner, otherwise it thinks it's cmd... + - run: bash .ci/run - - uses: actions/upload-artifact@v3 + - if: matrix.platform == 'ubuntu-latest' # no need to compute coverage for other platforms + uses: actions/upload-artifact@v3 with: name: .coverage.mypy_${{ matrix.platform }}_${{ matrix.python-version }} path: .coverage.mypy/ @@ -62,7 +69,7 @@ jobs: - uses: actions/setup-python@v4 with: - python-version: '3.7' + python-version: '3.8' - uses: actions/checkout@v3 with: diff --git a/Makefile b/Makefile index c22f6de..d31c4ea 100644 --- a/Makefile +++ b/Makefile @@ -11,4 +11,4 @@ doc: cog ## Update files using cog.py cog: orgparse/__init__.py orgparse/__init__.py: README.rst - cd orgparse && cog.py -r __init__.py + cd src/orgparse && cog.py -r __init__.py diff --git a/conftest.py b/conftest.py new file mode 100644 index 0000000..466da24 --- /dev/null +++ b/conftest.py @@ -0,0 +1,38 @@ +# this is a hack to monkey patch pytest so it handles tests inside namespace packages without __init__.py properly +# without it, pytest can't discover the package root for some reason +# also see https://github.com/karlicoss/pytest_namespace_pkgs for more + +import pathlib +from typing import Optional + +import _pytest.main +import _pytest.pathlib + +# we consider all dirs in repo/ to be namespace packages +root_dir = pathlib.Path(__file__).absolute().parent.resolve() / 'src' +assert root_dir.exists(), root_dir + +# TODO assert it contains package name?? maybe get it via setuptools.. + +namespace_pkg_dirs = [str(d) for d in root_dir.iterdir() if d.is_dir()] + +# resolve_package_path is called from _pytest.pathlib.import_path +# takes a full abs path to the test file and needs to return the path to the 'root' package on the filesystem +resolve_pkg_path_orig = _pytest.pathlib.resolve_package_path +def resolve_package_path(path: pathlib.Path) -> Optional[pathlib.Path]: + result = path # search from the test file upwards + for parent in result.parents: + if str(parent) in namespace_pkg_dirs: + return parent + raise RuntimeError("Couldn't determine path for ", path) +_pytest.pathlib.resolve_package_path = resolve_package_path + + +# without patching, the orig function returns just a package name for some reason +# (I think it's used as a sort of fallback) +# so we need to point it at the absolute path properly +# not sure what are the consequences.. maybe it wouldn't be able to run against installed packages? not sure.. +search_pypath_orig = _pytest.main.search_pypath +def search_pypath(module_name: str) -> str: + return str(root_dir) +_pytest.main.search_pypath = search_pypath diff --git a/pytest.ini b/pytest.ini index 28d084d..20c3704 100644 --- a/pytest.ini +++ b/pytest.ini @@ -8,3 +8,6 @@ addopts = # otherwise it won't discover doctests --doctest-modules + + # show all test durations (unless they are too short) + --durations=0 diff --git a/setup.py b/setup.py index 5f02a7d..5d500b1 100644 --- a/setup.py +++ b/setup.py @@ -4,10 +4,9 @@ def main(): - pkg = 'orgparse' - subpkgs = find_namespace_packages('.', include=(pkg + '.*',)) + pkgs = find_namespace_packages('src') + pkg = min(pkgs) - import orgparse setup( name=pkg, use_scm_version={ @@ -18,22 +17,24 @@ def main(): zip_safe=False, - packages=[pkg, *subpkgs], + packages=pkgs, + package_dir={'': 'src'}, package_data={ pkg: ['py.typed'], # todo need the rest as well?? 'orgparse.tests.data': ['*.org'], }, - author=orgparse.__author__, + author='Takafumi Arakaki, Dmitrii Gerasimov', author_email='aka.tkf@gmail.com', maintainer='Dima Gerasimov (@karlicoss)', maintainer_email='karlicoss@gmail.com', url='https://github.com/karlicoss/orgparse', - license=orgparse.__license__, + license='BSD License', description='orgparse - Emacs org-mode parser in Python', - long_description=orgparse.__doc__, + # TODO add it back later, perhaps via ast? + # long_description=orgparse.__doc__, keywords='org org-mode emacs', classifiers=[ diff --git a/orgparse/__init__.py b/src/orgparse/__init__.py similarity index 98% rename from orgparse/__init__.py rename to src/orgparse/__init__.py index e3f45cb..416a3b7 100644 --- a/orgparse/__init__.py +++ b/src/orgparse/__init__.py @@ -113,8 +113,6 @@ from .node import parse_lines, OrgEnv, OrgNode # todo basenode?? -__author__ = 'Takafumi Arakaki, Dmitrii Gerasimov' -__license__ = 'BSD License' __all__ = ["load", "loads", "loadi"] diff --git a/orgparse/date.py b/src/orgparse/date.py similarity index 100% rename from orgparse/date.py rename to src/orgparse/date.py diff --git a/orgparse/extra.py b/src/orgparse/extra.py similarity index 100% rename from orgparse/extra.py rename to src/orgparse/extra.py diff --git a/orgparse/inline.py b/src/orgparse/inline.py similarity index 100% rename from orgparse/inline.py rename to src/orgparse/inline.py diff --git a/orgparse/node.py b/src/orgparse/node.py similarity index 100% rename from orgparse/node.py rename to src/orgparse/node.py diff --git a/orgparse/py.typed b/src/orgparse/py.typed similarity index 100% rename from orgparse/py.typed rename to src/orgparse/py.typed diff --git a/orgparse/tests/__init__.py b/src/orgparse/tests/__init__.py similarity index 100% rename from orgparse/tests/__init__.py rename to src/orgparse/tests/__init__.py diff --git a/orgparse/tests/data/00_simple.org b/src/orgparse/tests/data/00_simple.org similarity index 100% rename from orgparse/tests/data/00_simple.org rename to src/orgparse/tests/data/00_simple.org diff --git a/orgparse/tests/data/00_simple.py b/src/orgparse/tests/data/00_simple.py similarity index 100% rename from orgparse/tests/data/00_simple.py rename to src/orgparse/tests/data/00_simple.py diff --git a/orgparse/tests/data/01_attributes.org b/src/orgparse/tests/data/01_attributes.org similarity index 100% rename from orgparse/tests/data/01_attributes.org rename to src/orgparse/tests/data/01_attributes.org diff --git a/orgparse/tests/data/01_attributes.py b/src/orgparse/tests/data/01_attributes.py similarity index 100% rename from orgparse/tests/data/01_attributes.py rename to src/orgparse/tests/data/01_attributes.py diff --git a/orgparse/tests/data/02_tree_struct.org b/src/orgparse/tests/data/02_tree_struct.org similarity index 100% rename from orgparse/tests/data/02_tree_struct.org rename to src/orgparse/tests/data/02_tree_struct.org diff --git a/orgparse/tests/data/02_tree_struct.py b/src/orgparse/tests/data/02_tree_struct.py similarity index 100% rename from orgparse/tests/data/02_tree_struct.py rename to src/orgparse/tests/data/02_tree_struct.py diff --git a/orgparse/tests/data/03_repeated_tasks.org b/src/orgparse/tests/data/03_repeated_tasks.org similarity index 100% rename from orgparse/tests/data/03_repeated_tasks.org rename to src/orgparse/tests/data/03_repeated_tasks.org diff --git a/orgparse/tests/data/03_repeated_tasks.py b/src/orgparse/tests/data/03_repeated_tasks.py similarity index 100% rename from orgparse/tests/data/03_repeated_tasks.py rename to src/orgparse/tests/data/03_repeated_tasks.py diff --git a/orgparse/tests/data/04_logbook.org b/src/orgparse/tests/data/04_logbook.org similarity index 100% rename from orgparse/tests/data/04_logbook.org rename to src/orgparse/tests/data/04_logbook.org diff --git a/orgparse/tests/data/04_logbook.py b/src/orgparse/tests/data/04_logbook.py similarity index 100% rename from orgparse/tests/data/04_logbook.py rename to src/orgparse/tests/data/04_logbook.py diff --git a/orgparse/tests/data/05_tags.org b/src/orgparse/tests/data/05_tags.org similarity index 100% rename from orgparse/tests/data/05_tags.org rename to src/orgparse/tests/data/05_tags.org diff --git a/orgparse/tests/data/05_tags.py b/src/orgparse/tests/data/05_tags.py similarity index 100% rename from orgparse/tests/data/05_tags.py rename to src/orgparse/tests/data/05_tags.py diff --git a/orgparse/tests/data/__init__.py b/src/orgparse/tests/data/__init__.py similarity index 100% rename from orgparse/tests/data/__init__.py rename to src/orgparse/tests/data/__init__.py diff --git a/orgparse/tests/test_data.py b/src/orgparse/tests/test_data.py similarity index 96% rename from orgparse/tests/test_data.py rename to src/orgparse/tests/test_data.py index 75e75e4..f315878 100644 --- a/orgparse/tests/test_data.py +++ b/src/orgparse/tests/test_data.py @@ -13,7 +13,8 @@ def load_data(path): """Load data from python file""" ns = {} # type: ignore - exec(Path(path).read_text(), ns) + # read_bytes() and compile hackery to avoid encoding issues (e.g. see 05_tags) + exec(compile(Path(path).read_bytes(), path, 'exec'), ns) return ns['data'] diff --git a/orgparse/tests/test_date.py b/src/orgparse/tests/test_date.py similarity index 100% rename from orgparse/tests/test_date.py rename to src/orgparse/tests/test_date.py diff --git a/orgparse/tests/test_hugedata.py b/src/orgparse/tests/test_hugedata.py similarity index 100% rename from orgparse/tests/test_hugedata.py rename to src/orgparse/tests/test_hugedata.py diff --git a/orgparse/tests/test_misc.py b/src/orgparse/tests/test_misc.py similarity index 100% rename from orgparse/tests/test_misc.py rename to src/orgparse/tests/test_misc.py diff --git a/orgparse/tests/test_rich.py b/src/orgparse/tests/test_rich.py similarity index 100% rename from orgparse/tests/test_rich.py rename to src/orgparse/tests/test_rich.py diff --git a/tox.ini b/tox.ini index 3adf49d..788cbb5 100644 --- a/tox.ini +++ b/tox.ini @@ -2,21 +2,39 @@ minversion = 3.7 # relies on the correct version of Python installed envlist = tests,mypy +# https://github.com/tox-dev/tox/issues/20#issuecomment-247788333 +# hack to prevent .tox from crapping to the project directory +toxworkdir={env:TOXWORKDIR_BASE:}{toxinidir}/.tox [testenv] -passenv = CI,CI_* +# TODO how to get package name from setuptools? +package_name = "orgparse" +passenv = +# useful for tests to know they are running under ci + CI + CI_* +# respect user's cache dirs to prevent tox from crapping into project dir + MYPY_CACHE_DIR + PYTHONPYCACHEPREFIX + +# note: --use-pep517 here is necessary for tox --parallel flag to work properly +# otherwise it seems that it tries to modify .eggs dir in parallel and it fails [testenv:tests] commands = - pip install -e .[testing] - python -m pytest --ignore-glob='**/_py3compat.py' orgparse {posargs} + {envpython} -m pip install --use-pep517 -e .[testing] + # posargs allow test filtering, e.g. tox ... -- -k test_name + {envpython} -m pytest \ + --pyargs {[testenv]package_name} \ + {posargs} + [testenv:mypy] commands = - pip install -e .[linting] - python -m mypy --install-types --non-interactive \ - orgparse \ - # txt report is a bit more convenient to view on CI - --txt-report .coverage.mypy \ - --html-report .coverage.mypy \ - {posargs} + {envpython} -m pip install --use-pep517 -e .[linting] + {envpython} -m mypy --install-types --non-interactive \ + -p {[testenv]package_name} \ + # txt report is a bit more convenient to view on CI + --txt-report .coverage.mypy \ + --html-report .coverage.mypy \ + {posargs}