Merge pull request #41 from umr-lops/revisit-ci

add workflows and adapt pyproj for pre-commit

.github/workflows/ci.yaml

@@ -0,0 +1,84 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  detect-skip-ci-trigger:
+    name: "Detect CI Trigger: [skip-ci]"
+    if: |
+      github.repository == 'umr-lops/xarray-safe-s1'
+      && (
+          github.event_name == 'push' || github.event_name == 'pull_request'
+      )
+    runs-on: ubuntu-latest
+    outputs:
+      triggered: ${{ steps.detect-trigger.outputs.trigger-found }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 2
+      - uses: xarray-contrib/ci-trigger@v1
+        id: detect-trigger
+        with:
+          keyword: "[skip-ci]"
+
+  ci:
+    name: ${{ matrix.os }} py${{ matrix.python-version }}
+    runs-on: ${{ matrix.os }}
+    needs: detect-skip-ci-trigger
+
+    if: needs.detect-skip-ci-trigger.outputs.triggered == 'false'
+
+    defaults:
+      run:
+        shell: bash -l {0}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10", "3.11", "3.12"]
+        os: ["ubuntu-latest", "macos-latest", "windows-latest"]
+
+    steps:
+      - name: Checkout the repository
+        uses: actions/checkout@v4
+        with:
+          # need to fetch all tags to get a correct version
+          fetch-depth: 0 # fetch all branches and tags
+
+      - name: Setup environment variables
+        run: |
+          echo "TODAY=$(date +'%Y-%m-%d')" >> $GITHUB_ENV
+
+          echo "CONDA_ENV_FILE=ci/requirements/environment.yaml" >> $GITHUB_ENV
+
+      - name: Setup micromamba
+        uses: mamba-org/setup-micromamba@v2
+        with:
+          environment-file: ${{ env.CONDA_ENV_FILE }}
+          environment-name: xarray-safe-s1-tests
+          cache-environment: true
+          cache-environment-key: "${{runner.os}}-${{runner.arch}}-py${{matrix.python-version}}-${{env.TODAY}}-${{hashFiles(env.CONDA_ENV_FILE)}}"
+          create-args: >-
+            python=${{matrix.python-version}}
+
+      - name: Install xarray-safe-s1
+        run: |
+          python -m pip install --no-deps -e .
+
+      - name: Import xarray-safe-s1
+        run: |
+          python -c "import safe_s1"
+
+      - name: Run tests
+        run: |
+          python -m pytest --cov=safe_s1

.github/workflows/publish.yml

@@ -9,14 +9,14 @@ jobs:
     name: Publish to PyPI
     runs-on: ubuntu-latest
     permissions:
-      contents: 'read'
-      id-token: 'write'
+      contents: "read"
+      id-token: "write"
     steps:
       - uses: actions/checkout@v4
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: '3.x'
+          python-version: "3.x"
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip build twine
@@ -32,4 +32,4 @@ jobs:
         with:
           password: ${{ secrets.pypi_token }}
           repository_url: https://upload.pypi.org/legacy/
-          verify_metadata: true
+          verify_metadata: true

.github/workflows/upstream-dev.yaml

@@ -0,0 +1,98 @@
+name: upstream-dev CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+  schedule:
+    - cron: "0 18 * * 0" # Weekly "On Sundays at 18:00" UTC
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  detect-test-upstream-trigger:
+    name: "Detect CI Trigger: [test-upstream]"
+    if: github.event_name == 'push' || github.event_name == 'pull_request'
+    runs-on: ubuntu-latest
+    outputs:
+      triggered: ${{ steps.detect-trigger.outputs.trigger-found }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 2
+      - uses: xarray-contrib/[email protected]
+        id: detect-trigger
+        with:
+          keyword: "[test-upstream]"
+
+  upstream-dev:
+    name: upstream-dev
+    runs-on: ubuntu-latest
+    needs: detect-test-upstream-trigger
+
+    if: |
+      always()
+      && github.repository == 'umr-lops/xarray-safe-s1'
+      && (
+        github.event_name == 'schedule'
+        || github.event_name == 'workflow_dispatch'
+        || needs.detect-test-upstream-trigger.outputs.triggered == 'true'
+        || contains(github.event.pull_request.labels.*.name, 'run-upstream')
+      )
+
+    defaults:
+      run:
+        shell: bash -l {0}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.12"]
+
+    steps:
+      - name: checkout the repository
+        uses: actions/checkout@v4
+        with:
+          # need to fetch all tags to get a correct version
+          fetch-depth: 0 # fetch all branches and tags
+
+      - name: set up conda environment
+        uses: mamba-org/setup-micromamba@v1
+        with:
+          environment-file: ci/requirements/environment.yaml
+          environment-name: tests
+          create-args: >-
+            python=${{ matrix.python-version }}
+            pytest-reportlog
+
+      - name: install upstream-dev dependencies
+        run: bash ci/install-upstream-dev.sh
+
+      - name: install the package
+        run: python -m pip install --no-deps -e .
+
+      - name: show versions
+        run: python -m pip list
+
+      - name: import
+        run: |
+          python -c 'import safe_s1'
+
+      - name: run tests
+        if: success()
+        id: status
+        run: |
+          python -m pytest -rf --report-log=pytest-log.jsonl
+
+      - name: report failures
+        if: |
+          failure()
+          && steps.tests.outcome == 'failure'
+          && github.event_name == 'schedule'
+        uses: xarray-contrib/issue-from-pytest-log@v1
+        with:
+          log-path: pytest-log.jsonl

ci/install-upstream-dev.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+conda remove -y --force cytoolz numpy xarray toolz python-dateutil
+python -m pip install \
+  -i https://pypi.anaconda.org/scientific-python-nightly-wheels/simple \
+  --no-deps \
+  --pre \
+  --upgrade \
+  numpy \
+  xarray
+python -m pip install --upgrade \
+  git+https://github.com/pytoolz/toolz \
+  git+https://github.com/dateutil/dateutil

ci/requirements/environment.yaml

@@ -2,7 +2,7 @@ name: xarray-safe-s1-tests
 channels:
   - conda-forge
 dependencies:
-  - python=3.10
+  - python
   # development
   - ipython
   - pre-commit
@@ -14,6 +14,7 @@ dependencies:
   # testing
   - pytest
   - pytest-reportlog
+  - pytest-cov
   - hypothesis
   - coverage
   # I/O
@@ -29,12 +30,9 @@ dependencies:
   - numpy
   - pandas
   - shapely
-  - datetime
   - geopandas
   - affine
   - pyproj
   # processing
-  - os
-  - io
   - lxml
   - jmespath

docs/api.rst

@@ -8,4 +8,3 @@ API reference
 
     .. autoclass:: Sentinel1Reader
         :members:
-

docs/conf.py

@@ -16,16 +16,16 @@
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
 extensions = [
-    'sphinx.ext.autosummary',
-    'sphinx.ext.autodoc',
+    "sphinx.ext.autosummary",
+    "sphinx.ext.autodoc",
     "myst_parser",
     "sphinx.ext.extlinks",
     "sphinx.ext.intersphinx",
     "IPython.sphinxext.ipython_directive",
     "IPython.sphinxext.ipython_console_highlighting",
-    'nbsphinx',
-    'jupyter_sphinx',
-    'sphinx.ext.napoleon'
+    "nbsphinx",
+    "jupyter_sphinx",
+    "sphinx.ext.napoleon",
 ]
 
 extlinks = {
@@ -36,9 +36,9 @@
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ["_templates"]
 
-html_static_path = ['_static']
+html_static_path = ["_static"]
 
-html_style = 'css/xsar.css'
+html_style = "css/xsar.css"
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
@@ -84,21 +84,21 @@
     "dask": ("https://docs.dask.org/en/latest", None),
     "xarray": ("https://docs.xarray.dev/en/latest/", None),
     "rasterio": ("https://rasterio.readthedocs.io/en/latest/", None),
-    "datatree": ("https://xarray-datatree.readthedocs.io/en/latest/", None)
+    "datatree": ("https://xarray-datatree.readthedocs.io/en/latest/", None),
 }
 
 html_theme_options = {
-    'navigation_depth': 4,  # FIXME: doesn't work as expeted: should expand side menu
-    'collapse_navigation': False # FIXME: same as above
+    "navigation_depth": 4,  # FIXME: doesn't work as expeted: should expand side menu
+    "collapse_navigation": False,  # FIXME: same as above
 }
 
 # If true, links to the reST sources are added to the pages.
 html_show_sourcelink = False
 
 nbsphinx_allow_errors = False
 
-nbsphinx_execute = 'always'
+nbsphinx_execute = "always"
 
 nbsphinx_timeout = 300
 
-today_fmt = '%b %d %Y at %H:%M'
+today_fmt = "%b %d %Y at %H:%M"

docs/index.rst

@@ -79,4 +79,4 @@ Last documentation build: |today|
 .. _xarray.Dataset: http://xarray.pydata.org/en/stable/generated/xarray.Dataset.html
 .. _`recommended installation`: installing.rst#recommended-packages
 .. _SAFE format: https://sentinel.esa.int/web/sentinel/user-guides/sentinel-1-sar/data-formats
-.. _jupyter notebook: https://jupyter.readthedocs.io/en/latest/running.html#running
+.. _jupyter notebook: https://jupyter.readthedocs.io/en/latest/running.html#running

docs/installing.rst

@@ -45,4 +45,4 @@ Pytest configuration
 Pytest uses a default configuration file (`config.yml`) in which we can found products paths to test.
 This configuration can be superseded by adding a local config file on the home directory :
 (`~/xarray-safe-s1/localconfig.yml`).
-In this file, testing files can be listed in the var `product_paths`.
+In this file, testing files can be listed in the var `product_paths`.

pyproject.toml

@@ -37,3 +37,36 @@ skip_gitignore = true
 float_to_top = true
 default_section = "THIRDPARTY"
 known_first_party = "safe_s1"
+
+[tool.coverage.report]
+show_missing = true
+exclude_lines = ["pragma: no cover", "if TYPE_CHECKING"]
+
+[tool.ruff.lint]
+ignore = [
+  "E402",  # module level import not at top of file
+  "E501",  # line too long - let black worry about that
+  "E731",  # do not assign a lambda expression, use a def
+  "UP038", # type union instead of tuple for isinstance etc
+]
+select = [
+  "F",   # Pyflakes
+  "E",   # Pycodestyle
+  "I",   # isort
+  "UP",  # Pyupgrade
+  "TID", # flake8-tidy-imports
+  "W",
+]
+extend-safe-fixes = [
+  "TID252", # absolute imports
+  "UP031",  # percent string interpolation
+]
+fixable = ["I", "TID252", "UP"]
+
+[tool.ruff.lint.isort]
+known-first-party = ["safe_s1"]
+known-third-party = ["xarray", "toolz", "construct"]
+
+[tool.ruff.lint.flake8-tidy-imports]
+# Disallow all relative imports.
+ban-relative-imports = "all"

safe_s1/__init__.py

@@ -1,12 +1,12 @@
 import traceback
-#import safe_s1
 from safe_s1.reader import Sentinel1Reader
+
 try:
     from importlib import metadata
-except ImportError: # for Python<3.8
+except ImportError:  # for Python<3.8
     import importlib_metadata as metadata
-try: 
+try:
     __version__ = metadata.version("xarray-safe-s1")
 except Exception:
-    print('trace',traceback.format_exc())
+    print("trace", traceback.format_exc())
     __version__ = "999"

safe_s1/config.yml

@@ -1,3 +1,3 @@
 # default data paths for tests
 product_paths:
-  - 'S1A_IW_GRDH_1SDV_20170907T103020_20170907T103045_018268_01EB76_Z010.SAFE'
+  - "S1A_IW_GRDH_1SDV_20170907T103020_20170907T103045_018268_01EB76_Z010.SAFE"