build,refactor: update development tooling and package configuration

.github/workflows/icon4py-qa.yml

@@ -18,22 +18,19 @@ jobs:
                 run: |
                     sudo apt-get update
                     sudo apt-get install libboost-all-dev
+                    curl -LsSf https://astral.sh/uv/install.sh | sh
             - name: Set up Python
               uses: actions/setup-python@v4
               with:
                   python-version: "3.10"
                   cache: 'pip'
                   cache-dependency-path: |
                     **/pyproject.toml
-                    **/base-requirements.txt
-                    **/base-requirements-dev.txt
-                    **/requirements.txt
-                    **/requirements-dev.txt
-            - name: Install all icon4py namespace packages
+                    uv.lock
+            - name: Install venv and icon4py namespace packages
               run: |
-                  python -m pip install --upgrade pip setuptools wheel
-                  python -m pip install -r ./requirements-dev.txt
-                  python -m pip list
+                  uv sync --extra all
             - name: Run checks
               run: |
+                  source .venv/bin/activate
                   pre-commit run

.pre-commit-config.yaml

@@ -1,57 +1,80 @@
+# NOTE: pre-commit runs all hooks from the root folder of the repository,
+# as regular git hooks do. Therefore, paths passed as arguments to the plugins
+# should always be relative to the root folder.
+
+default_stages: [pre-commit, pre-push]
+default_language_version:
+  python: python3.10
+minimum_pre_commit_version: 3.8.0
+files: |
+  (?x)^(
+      tools/.*|
+      model/common/.*
+  )$
+
 repos:
-  - repo: local
-    hooks:
-      - id: run-common-precommit
-        name: Run Model Common Pre-commit
-        entry: pre-commit run --config model/common/.pre-commit-config.yaml --all-files
-        language: system
-        pass_filenames: false
-        always_run: true
-
-      - id: run-driver-precommit
-        name: Run Model Driver Pre-commit
-        entry: pre-commit run --config model/driver/.pre-commit-config.yaml --all-files
-        language: system
-        pass_filenames: false
-        always_run: true
-
-      - id: run-atmosphere-advection-precommit
-        name: Run Model Atmosphere Advection Pre-commit
-        entry: pre-commit run --config model/atmosphere/advection/.pre-commit-config.yaml --all-files
-        language: system
-        pass_filenames: false
-        always_run: true
-
-      - id: run-atmosphere-diffusion-precommit
-        name: Run Model Atmosphere Diffusion Pre-commit
-        entry: pre-commit run --config model/atmosphere/diffusion/.pre-commit-config.yaml --all-files
-        language: system
-        pass_filenames: false
-        always_run: true
-
-      - id: run-atmosphere-dycore-precommit
-        name: Run Model Atmosphere Dycore Pre-commit
-        entry: pre-commit run --config model/atmosphere/dycore/.pre-commit-config.yaml --all-files
-        language: system
-        pass_filenames: false
-        always_run: true
-
-      - id: run-atmosphere-subgrid_scale_physics-microphysics-precommit
-        name: Run Model Atmosphere Subgrid_scale_physics Microphysics Pre-commit
-        entry: pre-commit run --config model/atmosphere/subgrid_scale_physics/microphysics/.pre-commit-config.yaml --all-files
-        language: system
-        pass_filenames: false
-        always_run: true
-
-      - id: run-tools-precommit
-        name: Run Tools Pre-commit
-        entry: pre-commit run --config tools/.pre-commit-config.yaml --all-files
-        language: system
-        pass_filenames: false
-        always_run: true
-
-  - repo: https://github.com/gauge-sh/tach-pre-commit
-    rev: v0.10.7
-    hooks:
-    - id: tach
-      name: Check inter-package dependencies
+- repo: meta
+  hooks:
+  - id: check-hooks-apply
+    stages: [manual]
+  - id: check-useless-excludes
+    stages: [manual]
+
+- repo: https://github.com/pre-commit/pre-commit-hooks
+  rev: v4.3.0
+  hooks:
+  - id: check-case-conflict
+  - id: check-merge-conflict
+  - id: check-shebang-scripts-are-executable
+  - id: check-symlinks
+  - id: check-yaml
+  - id: debug-statements
+  - id: destroyed-symlinks
+
+# - repo: https://github.com/macisamuele/language-formatters-pre-commit-hooks
+#   rev: v2.6.0
+#   hooks:
+#   - id: pretty-format-ini
+#     args: [--autofix]
+#   - id: pretty-format-toml
+#     args: [--autofix]
+#   - id: pretty-format-yaml
+#     args: [--autofix, --preserve-quotes, --indent, "2"]
+
+- repo: https://github.com/pre-commit/mirrors-prettier
+  rev: v3.0.0-alpha.4
+  hooks:
+  - id: prettier
+    types_or: [markdown, json]
+
+- repo: https://github.com/Lucas-C/pre-commit-hooks
+  rev: v1.5.5
+  hooks:
+  - id: insert-license
+    name: add license for all ICON4Py Python source files
+    types: [python]
+    args: [--comment-style, "|#|", --license-filepath, HEADER.txt, --fuzzy-match-generates-todo]
+
+- repo: https://github.com/astral-sh/ruff-pre-commit
+  rev: v0.2.2
+  hooks:
+  - id: ruff
+    args: [--fix]
+  - id: ruff-format
+
+- repo: https://github.com/gauge-sh/tach-pre-commit
+  rev: v0.16.0
+  hooks:
+  - id: tach
+    name: Check inter-package dependencies
+
+# - repo: local
+#   hooks:
+#   - id: mypy
+#     name: mypy static type checker
+#     entry: bash -c 'mypy tools/src/icon4pytools model/common/src/icon4py/model/common'
+#     language: system
+#     types_or: [python, pyi]
+#     pass_filenames: false
+#     require_serial: true
+#     stages: [pre-commit]

.python-version

@@ -0,0 +1 @@
+3.10

README.md

@@ -1,95 +1,63 @@
 [![Open in Gitpod](https://img.shields.io/badge/Gitpod-ready--to--code-908a85?logo=gitpod)](https://gitpod.io/#https://github.com/C2SM/icon4py)
+[![Nox](https://img.shields.io/badge/%F0%9F%A6%8A-Nox-D85E00.svg)](https://github.com/wntrblm/nox)
+
 
 # ICON4Py
 
-ICON4Py hosts Python implementations of various components from the ICON climate and weather model. Additionally, it includes icon4pytools, a collection of command-line interfaces (CLIs), and utilities required for the integration of ICON4Py code into the ICON Fortran model. ICON4Py leverages [GT4Py](https://github.com/GridTools/gt4py) to ensure efficient and performance portable implementations of these components.
+This repository hosts a work-in-progress Python implementation of the ICON climate and weather model. Additionally, it includes `icon4pytools`, a collection of command-line interfaces (CLIs), and utilities required for the integration of ICON4Py code into the ICON Fortran model. ICON4Py leverages [GT4Py](https://github.com/GridTools/gt4py) to ensure efficient and performance portable implementations of these components.
 
 ## Project Structure
 
-The repository is organized into directories, each containing independent Python namespace packages for different ICON components or utility packages. These packages can be installed independently. Since these packages are not available from a package repository (yet), you need to specify the location of dependencies within this repository. This can be done by installing the required dependencies first. Refer to the [Installation instructions](#installation-instructions) below.
+The repository is organized as a _monorepo_, where various ICON model components and utilities are developed as independent Python namespace packages in subfolders. An `icon4py` Python package is defined at the root folder with the purpose to collect specific versions of the different components as package dependencies. The component can also be installed independently, although since they are not (yet) available from a package repository, they need to be installed from their specific location within this repository.
 
 ## License
 
 ICON4Py is licensed under the terms of the BSD-3-Clause.
 
 ## Installation instructions
-### Dependencies
-A minimal installation of ICON4Py needs 
-- Python 3.10
-- boost >= 1.85.0
 
-You can install all packages at once by using the provided `requirements.txt` or `requirements-dev.txt` files in the root of the repository. For example:
-The `-dev.txt` file installs ICON4Py packages and GT4Py in editable mode, such that source changes are immediatly picked up and used in the virtual environment. 
-```bash
-# Clone the repository
-git clone [email protected]:C2SM/icon4py.git
-cd icon4py
-
-# Create a (Python 3.10) virtual environment (usually at `.venv`)
-python3.10 -m venv .venv
+Since this project is still in a highly experimental state, it is not yet available as a regular Python distribution project through PyPI. The installation procedure comprises cloning the [https://github.com/C2SM/icon4py](https://github.com/C2SM/icon4py) repository and install it in a _venv_ using the following development workflow.  
 
-# Activate the virtual environment and make sure that 'wheel' is installed
-source .venv/bin/activate
-pip install --upgrade wheel pip setuptools
+### System dependencies
 
-# Install all the ICON4Py packages and its dependencies
-# External dependencies would be checked out at './_external_src'
-pip install --src _external_src -r requirements-dev.txt
+ICON4Py requires **_Python >= 3.10_** and **_boost >= 1.85.0_**, and uses the `uv` tool to manage the development workflow. `uv` is a versatile tool which bundles together functionality from different applications: it can work as a _fast_ Python package manager (like `pip`), as a dependency version exporter (like `pip-tools`), as a Python application runner (like `pipx`) or as a full project development manager (like `hatch`). 
+`uv` can be installed in different ways (check its [installation instructions](https://docs.astral.sh/uv/getting-started/installation/)), like using the recommended standalone installer:
 
-# Finally, check that everything works
-pytest -v
+```bash
+$ curl -LsSf https://astral.sh/uv/install.sh | sh 
 ```
 
-The `--src _external_src` option tells `pip` to use a specific folder as the base path for checked out sources, which is very convenient for development tasks involving changes in external dependencies like `gt4py`. For convenience, `./_external_src` has been already added to the repository `.gitignore`.
-
-You can also use [tox](https://tox.wiki/en/latest/) for the automatic installation of all packages in development mode in a single step:
+### ICON4Py development environment
 
+Once `uv` is installed in your system, it is enough to clone this repository and let `uv` handling the installation of the development environment.
 
 ```bash
 # Clone the repository
 git clone [email protected]:C2SM/icon4py.git
 cd icon4py
 
-# Use tox to create and set up a development environment (usually at `.venv`) in verbose mode
-pip install tox
-python -m tox -vv -e dev --devenv .venv
+# Let uv create the development environment at `.venv`.
+# The `--extra all` option tells uv to install all optional
+# dependencies of icon4py so it is not strictly necessary
+uv sync --extra all
 
-# Activate the virtual environment and check that everything works
+# Activate the virtual environment and start writing code!
 source .venv/bin/activate
-pytest -v
 ```
 
+The new _venv_ is a standard Python virtual environment preconfigured with all necessary runtime and development dependencies. Additionally, all icon4py subpackages are installed in editable mode, allowing for seamless development and testing.
 
+To install new packages, use the `uv pip` subcommand, which emulates the `pip` interface and is generally much faster. Alternatively, the standard `pip` command is also available within the venv, although using `pip` directly is slower and not recommended.
 
-### Installation of specific subpackages
-
-In case you only want to install a specific subpackage, use the actual subpackage `requirements.txt` or `requirements-dev.txt` files.
+The `pyproject.toml` file at the root folder contains both the definition of the `icon4py` Python distribution package and the settings of the development tools used in this project, most notably `uv`, `ruff`, `mypy` and `pytest`. It also contains _dependency groups_ (see [PEP 735](https://peps.python.org/pep-0735/) for further reference) with the development requirements listed in different groups (`build`, `docs`, `lint`, `test`, `typing`, ...) and collected together in the general `dev` group which gets installed by default by `uv`.
 
-For example:
 
-```bash
-# Clone the repository
-git clone [email protected]:C2SM/icon4py.git
-cd icon4py
-
-# Create a (Python 3.10) virtual environment (usually at `.venv`)
-python3.10 -m venv .venv
-
-# Activate the virtual environment and make sure that 'wheel' is installed
-source .venv/bin/activate
-pip install --upgrade wheel
-
-# Install a specific ICON4Py subpackage and its dependencies
-cd _SUBPACKAGE_  # where _SUBPACKAGE_ in model/atmosphere/dycore | tools | ...
-pip install -r requirements-dev.txt
+## Development instructions
 
-# or in the case of there being a pyproject.toml file
-pip install .
-```
+By following the installation instructions above, the source files are imported directly by the Python interpreter meaning that any code change is available and executed by the interpreter.
 
-## Development instructions
+To add new dependencies to the project, either core/optional run-time or development-only dependencies, it is possible to use the `uv` cli direcly or to modify by hand the appropriate tables in the corresponding `pyproject.toml` (check `uv` documentation for more information [https://docs.astral.sh/uv/concepts/projects/dependencies/](https://docs.astral.sh/uv/concepts/projects/dependencies/)).
 
-After following the installation instructions above using the development requirements (`*-dev.txt` files), an _editable_ installation of all the packages will be active in the virtual environment. In this mode, code changes are immediately visible since source files are imported directly by the Python interpreter.
 
 ### Code quality checks
 
@@ -115,17 +83,18 @@ pytest -v
 pytest -v path/to/test/folder
 ```
 
-Nonetheless, we also recommended to use `tox` to run the complete test suite:
+`nox` is recommended for running comprehensive test suites across multiple Python versions and configurations, mirroring the setup used in the CI pipeline.
 
 ```bash
-# Run test suite in the default environment
-tox
+# List all available test sessions (colored items are the default sessions)
+nox -l
 
-# Run test suite in a specific environment (use `tox -a` to see list of envs)
-tox -e py310
-```
+# Run all parametrized cases of a session
+nox -s 'test_common'
 
-The default `tox` environment is configured to generate HTML test coverage reports in `_reports/coverage_html/`.
+# Run a test session for a specific python version and parameter value
+nox -s 'test_atmosphere_advection-3.10(datatest=True)'
+```
 
 ### Benchmarking
 

ci/base.yml

@@ -26,19 +26,17 @@ variables:
     DOCKERFILE: ci/docker/base.Dockerfile
     # change to 'always' if you want to rebuild, even if target tag exists already (if-not-exists is the default, i.e. we could also skip the variable)
     CSCS_REBUILD_POLICY: if-not-exists
-    DOCKER_BUILD_ARGS: '["ARCH=$ARCH", "HPC_SDK_VERSION=$HPC_SDK_VERSION", "HPC_SDK_NAME=$HPC_SDK_NAME", "CUPY_PACKAGE=$CUPY_PACKAGE", "PYVERSION=$PYVERSION", "CI_PROJECT_DIR=$CI_PROJECT_DIR"]'
+    DOCKER_BUILD_ARGS: '["ARCH=$ARCH", "HPC_SDK_VERSION=$HPC_SDK_VERSION", "HPC_SDK_NAME=$HPC_SDK_NAME", "PYVERSION=$PYVERSION", "CI_PROJECT_DIR=$CI_PROJECT_DIR"]'
 build_baseimage_x86_64:
   extends: [.container-builder-cscs-zen2, .build_baseimage]
   variables:
     HPC_SDK_VERSION: 22.11
     HPC_SDK_NAME: "nvhpc_2022_2211_Linux_${ARCH}_cuda_11.8"
-    CUPY_PACKAGE: cupy-cuda11x
 build_baseimage_aarch64:
   extends: [.container-builder-cscs-gh200, .build_baseimage]
   variables:
     HPC_SDK_VERSION: 24.5
     HPC_SDK_NAME: "nvhpc_2024_245_Linux_${ARCH}_cuda_12.4"
-    CUPY_PACKAGE: cupy-cuda12x
   # TODO: re-enable CI job when Todi is back in operational state
   when: manual
 
@@ -60,7 +58,6 @@ build_image_aarch64:
   timeout: 8h
   image: $CSCS_REGISTRY_PATH/public/$ARCH/icon4py/icon4py-ci:$CI_COMMIT_SHA-$PYVERSION
   before_script:
-    - python -c "import cupy"
     - cd /icon4py
     - ls "${TEST_DATA_PATH}"
   variables:
@@ -81,6 +78,7 @@ build_image_aarch64:
   variables:
     CSCS_ADDITIONAL_MOUNTS: '["/project/d121/icon4py/ci/testdata:$TEST_DATA_PATH"]'
     HPC_SDK_PATH: "/opt/nvidia/hpc_sdk/Linux_${ARCH}/22.11"
+    ICON4PY_NOX_UV_CUSTOM_SESSION_EXTRAS: "cuda11"
 .test_template_aarch64:
   extends: [.container-runner-todi-gh200, .test_template]
   needs: [build_image_aarch64]
@@ -93,3 +91,4 @@ build_image_aarch64:
     # Another problem, observed in test stage, is that gpu tests hang in combination with CUDA MPS,
     # when high test parallelism is used.
     NUM_PROCESSES: 16
+    ICON4PY_NOX_UV_CUSTOM_SESSION_EXTRAS: "cuda12"

ci/benchmark.yml

@@ -5,7 +5,7 @@ include:
   stage: benchmark
   script:
     # force execution of tests where validation is expected to fail, because the reason for failure is wrong numpy reference
-    - tox -r -e run_benchmarks -c model/ -- --backend=$BACKEND --grid=$GRID --runxfail
+    - nox -s benchmark_model-3.10  -- --backend=$BACKEND --grid=$GRID --runxfail
   parallel:
     matrix:
       - BACKEND: [gtfn_cpu, gtfn_gpu]