Make environment files management consistent between pip and conda

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,6 @@
+<!-- Feel free to remove check-list items that aren't relevant to your change -->
+
+ - [ ] Resolves #xxx,
+ - [ ] Tests added, otherwise issue #xxx opened,
+ - [ ] New optional dependencies added to both `dev-environment.yml` and `setup.cfg`,
+ - [ ] Fully documented, including `api/*.md` for new API.

.github/scripts/generate_pip_deps_from_conda.py

@@ -0,0 +1,148 @@
+#!/usr/bin/env python3
+"""
+(Copied from pandas: https://github.com/pandas-dev/pandas/blob/main/scripts/generate_pip_deps_from_conda.py)
+Convert the conda environment.yml to the pip requirements-dev.txt,
+or check that they have the same packages (for the CI)
+
+Usage:
+
+    Generate `requirements-dev.txt`
+    $ python scripts/generate_pip_deps_from_conda.py
+
+    Compare and fail (exit status != 0) if `requirements-dev.txt` has not been
+    generated with this script:
+    $ python scripts/generate_pip_deps_from_conda.py --compare
+"""
+import argparse
+import pathlib
+import re
+import sys
+
+if sys.version_info >= (3, 11):
+    import tomllib
+else:
+    import tomli as tomllib
+import yaml
+
+EXCLUDE = {"python"}
+REMAP_VERSION = {"tzdata": "2022.1"}
+RENAME = {
+    "pytables": "tables",
+    "psycopg2": "psycopg2-binary",
+    "dask-core": "dask",
+    "seaborn-base": "seaborn",
+    "sqlalchemy": "SQLAlchemy",
+}
+
+
+def conda_package_to_pip(package: str):
+    """
+    Convert a conda package to its pip equivalent.
+
+    In most cases they are the same, those are the exceptions:
+    - Packages that should be excluded (in `EXCLUDE`)
+    - Packages that should be renamed (in `RENAME`)
+    - A package requiring a specific version, in conda is defined with a single
+      equal (e.g. ``pandas=1.0``) and in pip with two (e.g. ``pandas==1.0``)
+    """
+    package = re.sub("(?<=[^<>])=", "==", package).strip()
+    print(package)
+
+    for compare in ("<=", ">=", "=="):
+        if compare in package:
+            pkg, version = package.split(compare)
+            if pkg in EXCLUDE:
+                return
+            if pkg in REMAP_VERSION:
+                return "".join((pkg, compare, REMAP_VERSION[pkg]))
+            if pkg in RENAME:
+                return "".join((RENAME[pkg], compare, version))
+
+    if package in EXCLUDE:
+        return
+
+    if package in RENAME:
+        return RENAME[package]
+
+    return package
+
+
+def generate_pip_from_conda(conda_path: pathlib.Path, pip_path: pathlib.Path, compare: bool = False) -> bool:
+    """
+    Generate the pip dependencies file from the conda file, or compare that
+    they are synchronized (``compare=True``).
+
+    Parameters
+    ----------
+    conda_path : pathlib.Path
+        Path to the conda file with dependencies (e.g. `environment.yml`).
+    pip_path : pathlib.Path
+        Path to the pip file with dependencies (e.g. `requirements-dev.txt`).
+    compare : bool, default False
+        Whether to generate the pip file (``False``) or to compare if the
+        pip file has been generated with this script and the last version
+        of the conda file (``True``).
+
+    Returns
+    -------
+    bool
+        True if the comparison fails, False otherwise
+    """
+    with conda_path.open() as file:
+        deps = yaml.safe_load(file)["dependencies"]
+
+    pip_deps = []
+    for dep in deps:
+        if isinstance(dep, str):
+            conda_dep = conda_package_to_pip(dep)
+            if conda_dep:
+                pip_deps.append(conda_dep)
+        elif isinstance(dep, dict) and len(dep) == 1 and "pip" in dep:
+            pip_deps.extend(dep["pip"])
+        else:
+            raise ValueError(f"Unexpected dependency {dep}")
+
+    header = (
+        f"# This file is auto-generated from {conda_path.name}, do not modify.\n"
+        "# See that file for comments about the need/usage of each dependency.\n\n"
+    )
+    pip_content = header + "\n".join(pip_deps) + "\n"
+
+    # add setuptools to requirements-dev.txt
+    with open(pathlib.Path(conda_path.parent, "pyproject.toml"), "rb") as fd:
+        meta = tomllib.load(fd)
+    for requirement in meta["build-system"]["requires"]:
+        if "setuptools" in requirement:
+            pip_content += requirement
+            pip_content += "\n"
+
+    if compare:
+        with pip_path.open() as file:
+            return pip_content != file.read()
+
+    with pip_path.open("w") as file:
+        file.write(pip_content)
+    return False
+
+
+if __name__ == "__main__":
+    argparser = argparse.ArgumentParser(description="convert (or compare) conda file to pip")
+    argparser.add_argument(
+        "--compare",
+        action="store_true",
+        help="compare whether the two files are equivalent",
+    )
+    args = argparser.parse_args()
+
+    conda_fname = "environment.yml"
+    pip_fname = "requirements.txt"
+    repo_path = pathlib.Path(__file__).parent.parent.parent.absolute()
+    res = generate_pip_from_conda(
+        pathlib.Path(repo_path, conda_fname),
+        pathlib.Path(repo_path, pip_fname),
+        compare=args.compare,
+    )
+    if res:
+        msg = f"`{pip_fname}` has to be generated with `{__file__}` after " f"`{conda_fname}` is modified.\n"
+        sys.stderr.write(msg)
+    sys.exit(res)

.github/workflows/python-app.yml

@@ -59,8 +59,8 @@ jobs:
       if: steps.cache.outputs.cache-hit != 'true'
       run: |
         mamba install pyyaml python=${{ matrix.python-version }}
-        pkgs_conda_base=`python .github/get_yml_env_nopy.py "environment.yml" --p "conda"`
-        pkgs_pip_base=`python .github/get_yml_env_nopy.py "environment.yml" --p "pip"`
+        pkgs_conda_base=`python .github/scripts/get_yml_env_nopy.py "environment.yml" --p "conda"`
+        pkgs_pip_base=`python .github/scripts/get_yml_env_nopy.py "environment.yml" --p "pip"`
         mamba install python=${{ matrix.python-version }} $pkgs_conda_base graphviz
         if [[ "$pkgs_pip_base" != "None" ]]; then
           pip install $pkgs_pip_base

.gitignore

@@ -135,8 +135,8 @@ dmypy.json
 # Emacs backup files
 *~
 
-# version file
-geoutils/version.py
+# Version file
+geoutils/_version.py
 
 # End of https://www.gitignore.io/api/python
 .vim/

.pre-commit-config.yaml

@@ -101,3 +101,13 @@ repos:
           rev: 2.0.0
           hooks:
                 - id: relint
+        - repo: local
+          hooks:
+                # Generate pip's requirements.txt from conda's environment.yml to ensure consistency
+                - id: pip-to-conda
+                  name: Generate pip dependency from conda
+                  language: python
+                  entry: .github/scripts/generate_pip_deps_from_conda.py
+                  files: ^(environment.yml|requirements.txt)$
+                  pass_filenames: false
+                  additional_dependencies: [tomli, pyyaml]

HOW_TO_RELEASE.md

@@ -1,11 +1,13 @@
 # How to issue a GeoUtils release
 
-## The easy way
+## GitHub and PyPI
 
-1. Change the version number in setup.py. It can be easily done from GitHub directly without a PR. The version number is important for PyPI as it will determine the file name of the wheel. A name can [never be reused](https://pypi.org/help/#file-name-reuse), even if a file or project have been deleted.
+### The easy way
+
+1. Change the version number in `setup.cfg`. It can be easily done from GitHub directly without a PR. The version number is important for PyPI as it will determine the file name of the wheel. A name can [never be reused](https://pypi.org/help/#file-name-reuse), even if a file or project have been deleted.
 
 2. Follow the steps to [create a new release](https://docs.github.com/en/repositories/releasing-projects-on-github/managing-releases-in-a-repository) on GitHub.
-Use the same release number and tag as in setup.py.
+Use the same release number and tag as in `setup.cfg`.
 
 An automatic GitHub action will start to push and publish the new release to PyPI.
 
@@ -18,7 +20,7 @@ An automatic GitHub action will start to push and publish the new release to PyP
 - Before releasing, you need to delete **both** the tag and the release of the previous release. If you release with the same tag without deletion, it will ignore your commit changing the version number, and PyPI will block the upload again. You're stuck in a circle.
 
 
-## The hard way
+### The hard way
 
  1. Go to your local main repository (not the fork) and ensure your master branch is synced:
        git checkout master
@@ -66,14 +68,24 @@ An automatic GitHub action will start to push and publish the new release to PyP
  15. Issue the release announcement!
 
 
-## conda-forge
-To update the conda-forge distribution of geoutils, a few steps have to be performed manually.
-The process **should** be automatic, but at the moment (September 2022), the automatic process does not function!
-conda-forge distributions work by having a "feedstock" version of the package, containing instructions on how to bundle it for conda.
-The geoutils feedstock is available at [https://github.com/conda-forge/geoutils-feedstock](https://github.com/conda-forge/geoutils-feedstock).
+## Conda-forge
+
+Conda-forge distributions work by having a "feedstock" version of the package, containing instructions on how to bundle it for conda.
+The GeoUtils feedstock is available at [https://github.com/conda-forge/geoutils-feedstock](https://github.com/conda-forge/geoutils-feedstock), and only accessible by maintainers.
+
+The process **should** largely be automatic, but at the moment (September 2023), the conda bot does not function for GeoUtils!
+
+### If the conda-forge bot works
+
+To update the conda-forge distribution of GeoUtils, very few steps should have to be performed manually. If the conda bot works, a PR will be opened at [https://github.com/conda-forge/geoutils-feedstock](https://github.com/conda-forge/geoutils-feedstock) within a day of publishing a new GitHub release.
+Assuming the dependencies have not changed, only two lines will be changed in the `meta.yaml` file of the feedstock: (i) the new version number and (ii) the new sha256 checksum for the GitHub-released package. Those will be updated automatically by the bot.
+
+However, if the dependencies or license need to be updated, this has to be done manually. Then, add the bot branch as a remote branch and push the dependency changes to `meta.yaml` (see additional info from conda bot for license).
+
+### If the conda-forge bot does not work
+
+In this case, the PR has to be opened manually, and the new version number and new sha256 checksum have to be updated manually as well.
 
-Assuming the dependencies have not changed, only two lines have to be changed in the `meta.yaml` file of the feedstock.
-The new version number has to be specified, as well as the new sha256 checksum for the package.
 The most straightforward way to obtain the new sha256 checksum is to run `conda-build` (see below) with the old checksum which will fail, and then copying the new hash of the "SHA256 mismatch: ..." error that arises!
 
 First, the geoutils-feedstock repo has to be forked on GitHub.
@@ -96,6 +108,9 @@ Then, follow these steps for `NEW_VERSION` (substitute with the actual version n
 
 >>> git push -u origin master
 ```
+
+An alternative solution to get the sha256sum is to run `sha256sum` on the release file downloaded from GitHub
+
 Now, a PR can be made from your personal fork to the upstream geoutils-feedstock.
 An automatic linter will say whether the updates conform to the syntax and a CI action will build the package to validate it.
 Note that you have to be a maintainer or have the PR be okayed by a maintainer for the CI action to run.

dev-environment.yml

@@ -13,26 +13,32 @@ dependencies:
   - xarray
   - rioxarray
 
-  # Development-specific
-  - autovizwidget
-  - graphviz
-  - sphinx
-  - myst-nb
-  - sphinx-autodoc-typehints
-  - sphinxcontrib-programoutput
-  - numpydoc
-  - flake8
+  # Development-specific, to mirror manually in setup.cfg [options.extras_require].
   - pip
+
+  # Optional dependencies
+  - scikit-image
+
+  # Test dependencies
   - pytest
   - pytest-xdist
   - pytest-lazy-fixture
+  - pyyaml
+  - flake8
   - pylint
-  - typing-extensions
+
+  # Doc dependencies
+  - sphinx
+  - sphinx-book-theme
   - sphinx-gallery
-  - scikit-image
-  - pyyaml
   - sphinx-design
-  - sphinx-book-theme
+  - sphinx-autodoc-typehints
+  - sphinxcontrib-programoutput
+  - autovizwidget
+  - graphviz
+  - myst-nb
+  - numpydoc
+  - typing-extensions
 
   - pip:
     - -e ./

doc/source/how_to_install.md

@@ -22,8 +22,8 @@ Once installed, the same commands can be run by simply replacing `conda` by `mam
 pip install geoutils
 ```
 
-```{note}
-Setting up GDAL and PROJ may require some extra steps, depending on your operating system and configuration.
+```{warning}
+Updating packages with `pip` (and sometimes `mamba`) can break your installation. If this happens, re-create an environment from scratch fixing directly all your dependencies.
 ```
 
 ## Installing for contributors

geoutils/__init__.py

@@ -7,7 +7,7 @@
 from geoutils.vector import Vector  # noqa
 
 try:
-    from geoutils.version import version as __version__  # noqa
+    from geoutils._version import __version__ as __version__  # noqa
 except ImportError:  # pragma: no cover
     raise ImportError(
         "geoutils is not properly installed. If you are "

geoutils/misc.py

@@ -39,7 +39,7 @@ def deprecator_func(func):  # type: ignore
         @functools.wraps(func)
         def new_func(*args, **kwargs):  # type: ignore
             # True if it should warn, False if it should raise an error
-            should_warn = removal_version is None or Version(removal_version) > Version(geoutils.version.version)
+            should_warn = removal_version is None or Version(removal_version) > Version(geoutils.__version__)
 
             # Add text depending on the given arguments and 'should_warn'.
             text = (
@@ -62,7 +62,7 @@ def new_func(*args, **kwargs):  # type: ignore
             if should_warn and removal_version is not None:
                 text += f" This functionality will be removed in version {removal_version}."
             elif not should_warn:
-                text += f" Current version: {geoutils.version.version}."
+                text += f" Current version: {geoutils.__version__}."
 
             if should_warn:
                 warnings.warn(text, category=DeprecationWarning, stacklevel=2)

pyproject.toml

@@ -1,3 +1,20 @@
+[build-system]
+# Minimum requirements for the build system to execute.
+requires = [
+    "setuptools>=42",
+    "setuptools_scm[toml]>=6.2",
+    "wheel",
+]
+build-backend = "setuptools.build_meta"
+
+# To write version to file
+[tool.setuptools_scm]
+write_to = "geoutils/_version.py"
+fallback_version = "0.0.1"
+
+[tool.black]
+target_version = ['py36']
+
 [tool.pytest.ini_options]
 addopts = "--doctest-modules"
 testpaths = [

requirements.txt

@@ -0,0 +1,14 @@
+# This file is auto-generated from environment.yml, do not modify.
+# See that file for comments about the need/usage of each dependency.
+
+geopandas>=0.12.0
+matplotlib
+pyproj
+rasterio>=1.3
+numpy
+scipy
+tqdm
+xarray
+rioxarray
+setuptools>=42
+setuptools_scm[toml]>=6.2