Merge greenline to main (#260)

merging of greenline with main

.github/workflows/icon4py-qa.yml

@@ -21,10 +21,14 @@ on:
         types: [submitted]
 
 jobs:
-    pre-commit-icon4py-model-common:
+    pre-commit-icon4py-model:
         runs-on: ubuntu-latest
         steps:
             - uses: actions/checkout@v3
+            -   name: Install system dependencies
+                run: |
+                    sudo apt-get update
+                    sudo apt-get install libboost-all-dev
             - name: Set up Python
               uses: actions/setup-python@v4
               with:
@@ -36,36 +40,22 @@ jobs:
                     **/base-requirements-dev.txt
                     **/requirements.txt
                     **/requirements-dev.txt
-            - name: Install icon4py-common
-              working-directory: model/common
+            - name: Install icon4py-model packages
+              working-directory: model
               run: |
                   python -m pip install --upgrade pip setuptools wheel
                   python -m pip install -r ./requirements-dev.txt
                   python -m pip list
             - name: Run checks icon4py-model-common
               run: |
                   pre-commit run --config model/common/.pre-commit-config.yaml --all-files
-
-    pre-commit-icon4py-model-atmosphere-dycore:
-        runs-on: ubuntu-latest
-        steps:
-            - uses: actions/checkout@v3
-            - name: Set up Python
-              uses: actions/setup-python@v4
-              with:
-                  python-version: "3.10"
-                  cache: 'pip'
-                  cache-dependency-path: |
-                    **/pyproject.toml
-                    **/base-requirements.txt
-                    **/base-requirements-dev.txt
-                    **/requirements.txt
-                    **/requirements-dev.txt
-            - name: Install icon4py-model-atmosphere-dycore
-              working-directory: model/atmosphere/dycore
+            - name: Run checks icon4py-model-driver
               run: |
-                  python -m pip install --upgrade pip setuptools wheel
-                  python -m pip install -r ./requirements-dev.txt
+                  pre-commit run --config model/driver/.pre-commit-config.yaml --all-files
             - name: Run checks icon4py-model-atmosphere-dycore
               run: |
                   pre-commit run --config model/atmosphere/dycore/.pre-commit-config.yaml --all-files
+            - name: Run checks icon4py-model-atmosphere-diffusion
+              run: |
+                  pre-commit run --config model/atmosphere/diffusion/.pre-commit-config.yaml --all-files
+

.github/workflows/icon4pytools-qa.yml

@@ -31,6 +31,10 @@ jobs:
         runs-on: ubuntu-latest
         steps:
             - uses: actions/checkout@v3
+            -   name: Install boost
+                run: |
+                    sudo apt-get update
+                    sudo apt-get install libboost-all-dev
             - name: Set up Python
               uses: actions/setup-python@v4
               with:

.gitignore

@@ -4,6 +4,8 @@ _local
 _external_src
 _reports
 tmp
+testdata
+simple_mesh*.nc
 
 ### GT4Py ####
 .gt_cache/

base-requirements-dev.txt

@@ -1,5 +1,7 @@
 # VCS
 -e git+https://github.com/GridTools/gt4py.git@main#egg=gt4py
+git+https://github.com/GridTools/serialbox#egg=serialbox&subdirectory=src/serialbox-python
+git+https://github.com/ghex-org/GHEX.git#subdirectory=bindings/python
 
 # PyPI
 bump2version>=1.0.1
@@ -13,13 +15,16 @@ flake8-eradicate>=1.3.0
 flake8-mutable>=1.2.0
 isort~=5.10
 mypy>=0.942
+typing-extensions==4.5.0
 pre-commit~=2.15
 pytest>=6.1
 pytest-benchmark>=4.0.0
 pytest-cache>=1.0
 pytest-cov>=2.8
 pytest-factoryboy>=2.0
 pytest-xdist[psutil]>=2.2
+pytest-mpi>=0.6
 setuptools>=40.8.0
 wheel>=0.37.1
 tox >= 3.25
+wget>=3.2

base-requirements.txt

@@ -1,2 +1,3 @@
 # VCS
 gt4py @ git+https://github.com/GridTools/gt4py.git@main
+pyghex @ git+https://github.com/boeschf/GHEX.git@pipify#subdirectory=bindings/python

model/README.md

@@ -5,7 +5,9 @@ This folder contains Python implementations for multiple ICON components.
 It includes the following packages:
 
 - `atmosphere/dycore`: Contains implementations of the dynamical core of the ICON model
+- `atmosphere/diffusion`: Contains the implementation of diffusion in the ICON model
 - `common`: Contains shared functionality that is required by multiple components.
+- `driver`: Contains the driving code for the model
 
 ## Installation Instructions
 
@@ -22,3 +24,47 @@ pip install -r requirements-dev.txt
 ```
 
 **Note**: For more information specific to each component, please refer to the README in their respective subfolders.
+
+### Testing
+
+See the repository [README](../README.md) for general information.
+
+#### Data dependent tests
+
+Some test depend on serialized data generated by a full model run. 
+Those test are marked with `pytest.mark.datatest` and are only run when the `--datatest` 
+option is specified. Note that due to `pytests` configuration discovery 
+you need to specify the base a package directory i.e. one that contains a `pyproject.toml` for the 
+commandline option to work.
+
+```bash
+
+pytest -v --datatest model/common
+pytest -v --datatest model/atmosphere/diffusion
+```
+
+#### Testing parallel code
+
+Tests for parallel codes using MPI are collected in specific subpackages of the model components test folders (e.g. `diffusion_tests/mpi_tests`). All parallel tests are marked with `@pytest.mark.mpi` and are skipped if the `--with-mpi` is not passed option is not passed to `pytest` In order to run them, you need a MPI installation on your system: On Debian-Linux do
+
+```bash
+sudo apt-get install libopenmpi-dev
+```
+
+or
+
+```bash
+sudo apt-get install mpich
+```
+
+on MacOs
+
+```bash
+brew install mpich
+```
+
+Then you can run the tests with
+
+```bash
+mpirun -np 2 pytest -v -s --with-mpi path/to/test/folder/mpi_tests
+```

model/atmosphere/diffusion/.bumpversion.cfg

@@ -0,0 +1,10 @@
+[bumpversion]
+current_version = 0.0.6
+parse = (?P<major>\d+)\.(?P<minor>\d+)(\.(?P<patch>\d+))?
+serialize =
+	{major}.{minor}.{patch}
+
+[bumpversion:file:src/icon4py/model/atmosphere/diffusion/__init__.py]
+parse = \"(?P<major>\d+)\.(?P<minor>\d+)(\.(?P<patch>\d+))?\"
+serialize =
+	{major}.{minor}.{patch}

model/atmosphere/diffusion/.flake8

@@ -0,0 +1,42 @@
+[flake8]
+# Some sane defaults for the code style checker flake8
+max-line-length = 100
+max-complexity = 15
+doctests = true
+extend-ignore =
+    # Do not perform function calls in argument defaults
+    B008,
+    # Public code object needs docstring
+    D1,
+    # Disable dargling errors by default
+    DAR,
+    # Whitespace before ':' (black formatter breaks this sometimes)
+    E203,
+    # Line too long (using Bugbear's B950 warning)
+    E501,
+    # Line break occurred before a binary operator
+    W503
+
+exclude =
+    .eggs,
+    .gt_cache,
+    .ipynb_checkpoints,
+    .tox,
+    _local_,
+    build,
+    dist,
+    docs,
+    _external_src,
+    tests/_disabled,
+    setup.py
+
+rst-roles =
+    py:mod, mod,
+    py:func, func,
+    py:data, data,
+    py:const, const,
+    py:class, class,
+    py:meth, meth,
+    py:attr, attr,
+    py:exc, exc,
+    py:obj, obj,

model/atmosphere/diffusion/.pre-commit-config.yaml

@@ -0,0 +1,114 @@
+# NOTE: pre-commit runs all hooks from the root folder of the repository,
+# as regular git hooks do. Therefore, paths passed as arguments to the plugins
+# should always be relative to the root folder.
+
+default_stages: [commit, push]
+default_language_version:
+  python: python3.10
+minimum_pre_commit_version: 2.20.0
+files: "model/atmosphere/diffusion/.*"
+
+repos:
+- repo: meta
+  hooks:
+  - id: check-hooks-apply
+    stages: [manual]
+  - id: check-useless-excludes
+    stages: [manual]
+
+- repo: https://github.com/asottile/setup-cfg-fmt
+  rev: v1.20.1
+  hooks:
+          # Run only manually because it deletes comments
+  - id: setup-cfg-fmt
+    name: format setup.cfg
+    stages: [manual]
+
+- repo: https://github.com/pre-commit/pre-commit-hooks
+  rev: v4.3.0
+  hooks:
+  - id: check-case-conflict
+  - id: check-merge-conflict
+  - id: check-shebang-scripts-are-executable
+  - id: check-symlinks
+  - id: check-yaml
+  - id: debug-statements
+  - id: destroyed-symlinks
+
+- repo: https://github.com/macisamuele/language-formatters-pre-commit-hooks
+  rev: v2.6.0
+  hooks:
+  - id: pretty-format-ini
+    args: [--autofix]
+  - id: pretty-format-toml
+    args: [--autofix]
+  - id: pretty-format-yaml
+    args: [--autofix, --preserve-quotes, --indent, "2"]
+
+- repo: https://github.com/pre-commit/mirrors-prettier
+  rev: v3.0.0-alpha.4
+  hooks:
+  - id: prettier
+    types_or: [markdown, json]
+
+- repo: https://github.com/Lucas-C/pre-commit-hooks
+  rev: v1.3.0
+  hooks:
+  - id: insert-license
+    name: add license for all ICON4Py Python source files
+    types: [python]
+    args: [--comment-style, "|#|", --license-filepath, model/.license_header.txt, --fuzzy-match-generates-todo]
+
+- repo: https://github.com/asottile/yesqa
+  rev: v1.3.0
+  hooks:
+  - id: yesqa
+
+- repo: https://github.com/psf/black
+  rev: '22.3.0'
+  hooks:
+  - id: black
+    name: black Python formatter
+    args: [--config, model/atmosphere/diffusion/pyproject.toml]
+
+- repo: https://github.com/asottile/blacken-docs
+  rev: v1.12.1
+  hooks:
+  - id: blacken-docs
+    name: black Python formatter for docstrings
+    additional_dependencies: [black==22.3.0]
+
+- repo: https://github.com/PyCQA/isort
+  rev: '5.12.0'
+  hooks:
+  - id: isort
+    args: [--config-root, model/atmosphere/diffusion/, --resolve-all-configs]
+
+- repo: https://github.com/PyCQA/flake8
+  rev: '4.0.1'
+  hooks:
+  - id: flake8
+    name: flake8 code style checks
+    additional_dependencies:
+    - darglint
+    - flake8-bugbear
+    - flake8-builtins
+    - flake8-debugger
+    - flake8-docstrings
+    - flake8-eradicate
+    - flake8-mutable
+    - pygments
+    args: [--config=model/atmosphere/diffusion/.flake8, model/atmosphere/diffusion/src/icon4py/]
+
+- repo: local
+  hooks:
+  - id: mypy
+    name: mypy static type checker
+    entry: bash -c 'echo mypy temporarily disabled'
+            #entry: bash -c 'cd model/atmosphere/dycore; mypy src/' --
+    language: system
+    types_or: [python, pyi]
+    always_run: true
+            #pass_filenames: false
+    require_serial: true
+    stages: [commit]

model/atmosphere/diffusion/README.md

@@ -0,0 +1,9 @@
+# icon4py-atmosphere-diffusion
+
+## Description
+
+Python port of ICON diffusion module.
+
+## Installation instructions
+
+Check the `README.md` at the root of the `model` folder for installation instructions.

model/atmosphere/diffusion/diffusion_tests/__init__.py

@@ -0,0 +1,12 @@
+# ICON4Py - ICON inspired code in Python and GT4Py
+#
+# Copyright (c) 2022, ETH Zurich and MeteoSwiss
+# All rights reserved.
+#
+# This file is free software: you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the
+# Free Software Foundation, either version 3 of the License, or any later
+# version. See the LICENSE.txt file at the top-level directory of this
+# distribution for a copy of the license or check <https://www.gnu.org/licenses/>.
+#
+# SPDX-License-Identifier: GPL-3.0-or-later