Mesh data model to ng vat mesh api

.cirrus.yml

@@ -1,5 +1,6 @@
 # Reference:
 #   - https://cirrus-ci.org/guide/writing-tasks/
+#   - https://cirrus-ci.org/guide/writing-tasks/#environment-variables
 #   - https://cirrus-ci.org/guide/tips-and-tricks/#sharing-configuration-between-tasks
 #   - https://cirrus-ci.org/guide/linux/
 #   - https://cirrus-ci.org/guide/macOS/
@@ -17,6 +18,16 @@ container:
 
 
 env:
+  # Skip specific tasks by name. Set to a non-empty string to skip.
+  SKIP_LINT_TASK: ""
+  SKIP_TEST_MINIMAL_TASK: ""
+  SKIP_TEST_FULL_TASK: ""
+  SKIP_GALLERY_TASK: ""
+  SKIP_DOCTEST_TASK: ""
+  SKIP_LINKCHECK_TASK: ""
+  # Skip task groups by type. Set to a non-empty string to skip.
+  SKIP_ALL_TEST_TASKS: ""
+  SKIP_ALL_DOC_TASKS: ""
   # Maximum cache period (in weeks) before forcing a new cache upload.
   CACHE_PERIOD: "2"
   # Increment the build number to force new cartopy cache upload.
@@ -35,25 +46,6 @@ env:
   IRIS_TEST_DATA_DIR: ${HOME}/iris-test-data
 
 
-#
-# Linting
-#
-lint_task:
-  auto_cancellation: true
-  name: "${CIRRUS_OS}: flake8 and black"
-  pip_cache:
-    folder: ~/.cache/pip
-    fingerprint_script:
-      - echo "${CIRRUS_TASK_NAME}"
-      - echo "$(date +%Y).$(($(date +%U) / ${CACHE_PERIOD})):${PIP_CACHE_BUILD} ${PIP_CACHE_PACKAGES}"
-  lint_script:
-    - pip list
-    - python -m pip install --retries 3 --upgrade ${PIP_CACHE_PACKAGES}
-    - pip list
-    - nox --session flake8
-    - nox --session black
-
-
 #
 # YAML alias for common linux test infra-structure.
 #
@@ -68,7 +60,7 @@ linux_task_template: &LINUX_TASK_TEMPLATE
     fingerprint_script:
       - wget --quiet https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh
       - echo "${CIRRUS_OS} $(sha256sum miniconda.sh)"
-      - echo "$(date +%Y).$(($(date +%U) / ${CACHE_PERIOD})):${CONDA_CACHE_BUILD}"
+      - echo "$(date +%Y).$(expr $(date +%U) / ${CACHE_PERIOD}):${CONDA_CACHE_BUILD}"
     populate_script:
       - bash miniconda.sh -b -p ${HOME}/miniconda
       - conda config --set always_yes yes --set changeps1 no
@@ -80,19 +72,49 @@ linux_task_template: &LINUX_TASK_TEMPLATE
     folder: ${HOME}/.local/share/cartopy
     fingerprint_script:
       - echo "${CIRRUS_OS}"
-      - echo "$(date +%Y).$(($(date +%U) / ${CACHE_PERIOD})):${CARTOPY_CACHE_BUILD}"
+      - echo "$(date +%Y).$(expr $(date +%U) / ${CACHE_PERIOD}):${CARTOPY_CACHE_BUILD}"
   nox_cache:
     folder: ${CIRRUS_WORKING_DIR}/.nox
     fingerprint_script:
       - echo "${CIRRUS_TASK_NAME}"
-      - echo "$(date +%Y).$(($(date +%U) / ${CACHE_PERIOD})):${NOX_CACHE_BUILD}"
+      - echo "$(date +%Y).$(expr $(date +%U) / ${CACHE_PERIOD}):${NOX_CACHE_BUILD}"
       - sha256sum ${CIRRUS_WORKING_DIR}/requirements/ci/py$(echo ${PY_VER} | tr -d ".").yml
 
 
+#
+# YAML alias for compute credits
+#
+compute_credits_template: &CREDITS_TEMPLATE
+  # Only use credits for non-DRAFT pull-requests to SciTools/iris master branch by collaborators
+  use_compute_credits: $CIRRUS_REPO_FULL_NAME == 'SciTools/iris' && $CIRRUS_USER_COLLABORATOR == 'true' && $CIRRUS_PR_DRAFT == 'false' && $CIRRUS_BASE_BRANCH == 'master' && $CIRRUS_PR != ''
+
+#
+# Linting
+#
+lint_task:
+  only_if: $SKIP_LINT_TASK == ""
+  << : *CREDITS_TEMPLATE
+  auto_cancellation: true
+  name: "${CIRRUS_OS}: flake8 and black"
+  pip_cache:
+    folder: ~/.cache/pip
+    fingerprint_script:
+      - echo "${CIRRUS_TASK_NAME}"
+      - echo "$(date +%Y).$(expr $(date +%U) / ${CACHE_PERIOD}):${PIP_CACHE_BUILD} ${PIP_CACHE_PACKAGES}"
+  lint_script:
+    - pip list
+    - python -m pip install --retries 3 --upgrade ${PIP_CACHE_PACKAGES}
+    - pip list
+    - nox --session flake8
+    - nox --session black
+
+
 #
 # Testing Minimal (Linux)
 #
-linux_minimal_task:
+test_minimal_task:
+  only_if: $SKIP_TEST_MINIMAL_TASK == "" && $SKIP_ALL_TEST_TASKS == ""
+  << : *CREDITS_TEMPLATE
   matrix:
     env:
       PY_VER: 3.6
@@ -115,7 +137,9 @@ linux_minimal_task:
 #
 # Testing Full (Linux)
 #
-linux_task:
+test_full_task:
+  only_if: $SKIP_TEST_FULL_TASK == "" && $SKIP_ALL_TEST_TASKS == ""
+  << : *CREDITS_TEMPLATE
   matrix:
     env:
       PY_VER: 3.6
@@ -148,6 +172,8 @@ linux_task:
 # Testing Documentation Gallery (Linux)
 #
 gallery_task:
+  only_if: $SKIP_GALLERY_TASK == "" && $SKIP_ALL_DOC_TASKS == ""
+  << : *CREDITS_TEMPLATE
   matrix:
     env:
       PY_VER: 3.8
@@ -176,6 +202,8 @@ gallery_task:
 # Testing Documentation (Linux)
 #
 doctest_task:
+  only_if: $SKIP_DOCTEST_TASK == "" && $SKIP_ALL_DOC_TASKS == ""
+  << : *CREDITS_TEMPLATE
   matrix:
     env:
       PY_VER: 3.8
@@ -209,7 +237,9 @@ doctest_task:
 #
 # Testing Documentation Link Check (Linux)
 #
-link_task:
+linkcheck_task:
+  only_if: $SKIP_LINKCHECK_TASK == "" && $SKIP_ALL_DOC_TASKS == ""
+  << : *CREDITS_TEMPLATE
   matrix:
     env:
       PY_VER: 3.8

docs/src/_static/theme_override.css

@@ -26,3 +26,17 @@ table.docutils td {
     word-wrap: break-word;
 }
 
+/* Used for very strong warning */
+#slim-red-box-message {
+  background: #ff0000;
+  box-sizing: border-box;
+  color: #ffffff;
+  font-weight: normal;
+  padding: 0.5em;
+}
+
+#slim-red-box-message a {
+  color: #ffffff;
+  font-weight: normal;  
+  text-decoration:underline;
+}

docs/src/_templates/layout.html

@@ -1,5 +1,27 @@
 {% extends "!layout.html" %}
 
+{# This uses blocks.  See: 
+     https://www.sphinx-doc.org/en/master/templating.html
+#}
+
+/*---------------------------------------------------------------------------*/
+
+{%- block document %}
+    {% if READTHEDOCS and rtd_version == 'latest' %}
+        <div id="slim-red-box-message">
+            You are viewing the <b>latest</b> unreleased documentation
+            <b>v{{ version }}</b>.  You may prefer a 
+            <a href="https://scitools-iris.readthedocs.io/en/stable/">stable</a>
+            version.
+        </div>
+        <p></p>
+    {%- endif %}
+
+    {{ super() }}
+{%- endblock %}
+
+/*-----------------------------------------------------z----------------------*/
+
 {% block menu %}
     {{ super() }}
 

docs/src/common_links.inc

@@ -3,6 +3,7 @@
 
 .. _black: https://black.readthedocs.io/en/stable/
 .. _.cirrus.yml: https://github.com/SciTools/iris/blob/master/.cirrus.yml
+.. _flake8: https://flake8.pycqa.org/en/stable/
 .. _.flake8.yml: https://github.com/SciTools/iris/blob/master/.flake8
 .. _cirrus-ci: https://cirrus-ci.com/github/SciTools/iris
 .. _conda: https://docs.conda.io/en/latest/
@@ -24,6 +25,7 @@
 .. _New Issue: https://github.com/scitools/iris/issues/new/choose
 .. _pull request: https://github.com/SciTools/iris/pulls
 .. _pull requests: https://github.com/SciTools/iris/pulls
+.. _Read the Docs: https://scitools-iris.readthedocs.io/en/latest/
 .. _readthedocs.yml: https://github.com/SciTools/iris/blob/master/requirements/ci/readthedocs.yml
 .. _SciTools: https://github.com/SciTools
 .. _sphinx: https://www.sphinx-doc.org/en/master/

docs/src/conf.py

@@ -43,6 +43,9 @@ def autolog(message):
     for item, value in os.environ.items():
         autolog("[READTHEDOCS] {} = {}".format(item, value))
 
+# This is the rtd reference to the version, such as: latest, stable, v3.0.1 etc
+# For local testing purposes this could be explicitly set latest or stable.
+rtd_version = os.environ.get("READTHEDOCS_VERSION")
 
 # -- Path setup --------------------------------------------------------------
 
@@ -131,7 +134,6 @@ def autolog(message):
     "custom_data_autodoc",
     "generate_package_rst",
 ]
-
 # -- panels extension ---------------------------------------------------------
 # See https://sphinx-panels.readthedocs.io/en/latest/
 
@@ -165,7 +167,7 @@ def autolog(message):
 # See https://sphinx-copybutton.readthedocs.io/en/latest/
 copybutton_prompt_text = ">>> "
 
-# sphinx.ext.todo configuration
+# sphinx.ext.todo configuration -----------------------------------------------
 # See https://www.sphinx-doc.org/en/master/usage/extensions/todo.html
 todo_include_todos = True
 
@@ -228,6 +230,8 @@ def autolog(message):
 }
 
 html_context = {
+    "rtd_version": rtd_version,
+    "version": version,
     "copyright_years": copyright_years,
     "python_version": build_python_version,
     # menu_links and menu_links_name are used in _templates/layout.html
@@ -296,7 +300,6 @@ def autolog(message):
     "ignore_pattern": r"__init__\.py",
 }
 
-
 # -----------------------------------------------------------------------------
 # Remove matplotlib agg warnings from generated doc when using plt.show
 warnings.filterwarnings(
@@ -306,7 +309,6 @@ def autolog(message):
     " non-GUI backend, so cannot show the figure.",
 )
 
-
 # -- numfig options (built-in) ------------------------------------------------
 # Enable numfig.
 numfig = True

docs/src/developers_guide/contributing_ci_tests.rst

@@ -5,9 +5,9 @@
 Continuous Integration (CI) Testing
 ===================================
 
-The `Iris`_ GitHub repository is configured to run checks on the code
-automatically when a pull request is created, updated or merged against
-Iris **master**.  The checks performed are:
+The `Iris`_ GitHub repository is configured to run checks against all its
+branches automatically whenever a pull request is created, updated or merged.
+The checks performed are:
 
 * :ref:`testing_cla`
 * :ref:`testing_cirrus`
@@ -18,29 +18,65 @@ Iris **master**.  The checks performed are:
 SciTools CLA Checker
 ********************
 
-A bot that checks the user who created the pull request has signed the
-**Contributor's License Agreement (CLA)**.  For more information on this this
-please see https://scitools.org.uk/organisation.html#governance
+A bot which checks that the GitHub author of the pull request has signed the
+**SciTools Contributor's License Agreement (CLA)**.  For more information on
+this please see https://scitools.org.uk/organisation.html#governance.
 
 
 .. _testing_cirrus:
 
 Cirrus-CI
 *********
 
-The unit and integration tests in Iris are an essential mechanism to ensure
+Iris unit and integration tests are an essential mechanism to ensure
 that the Iris code base is working as expected.  :ref:`developer_running_tests`
-may be run manually but to ensure the checks are performed a
-continuous integration testing tool named `cirrus-ci`_ is used.
+may be performed manually by a developer locally. However Iris is configured to
+use the `cirrus-ci`_ service for automated Continuous Integration (CI) testing.
 
-A `cirrus-ci`_ configuration file named `.cirrus.yml`_
-is in the Iris repository which tells Cirrus-CI what commands to run.  The
-commands include retrieving the Iris code base and associated test files using
-conda and then running the tests.  `cirrus-ci`_ allows for a matrix of tests to
-be performed to ensure that all expected variations test successfully.
+The `cirrus-ci`_ configuration file  `.cirrus.yml`_ in the root of the Iris repository
+defines the tasks to be performed by `cirrus-ci`_. For further details
+refer to the `Cirrus-CI Documentation`_. The tasks performed during CI include:
+
+* linting the code base and ensuring it adheres to the `black`_ format
+* running the system, integration and unit tests for Iris
+* ensuring the documentation gallery builds successfully
+* performing all doc-tests within the code base
+* checking all URL references within the code base and documentation are valid
+
+The above `cirrus-ci`_ tasks are run automatically against all `Iris`_ branches
+on GitHub whenever a pull request is submitted, updated or merged. See the
+`Cirrus-CI Dashboard`_ for details of recent past and active Iris jobs.
+
+.. _skipping Cirrus-CI tasks:
+
+Skipping Cirrus-CI Tasks
+------------------------
+
+As a developer you may wish to not run all the CI tasks when you are actively
+developing e.g., you are writing documentation and there is no need for linting,
+or long running compute intensive testing tasks to be executed.
+
+As a convenience, it is possible to easily skip one or more tasks by setting
+the appropriate environment variable within the `.cirrus.yml`_ file to a
+**non-empty** string:
+
+* ``SKIP_LINT_TASK`` to skip `flake8`_ linting and `black`_ formatting
+* ``SKIP_TEST_MINIMAL_TASK`` to skip restricted unit and integration testing
+* ``SKIP_TEST_FULL_TASK`` to skip full unit and integration testing
+* ``SKIP_GALLERY_TASK`` to skip building the documentation gallery
+* ``SKIP_DOCTEST_TASK`` to skip running the documentation doc-tests
+* ``SKIP_LINKCHECK_TASK`` to skip checking for broken documentation URL references
+* ``SKIP_ALL_TEST_TASKS`` which is equivalent to setting ``SKIP_TEST_MINIMAL_TASK`` and ``SKIP_TEST_FULL_TASK``
+* ``SKIP_ALL_DOC_TASKS`` which is equivalent to setting ``SKIP_GALLERY_TASK``, ``SKIP_DOCTEST_TASK``, and ``SKIP_LINKCHECK_TASK``
+
+e.g., to skip the linting task, the following are all equivalent::
+
+   SKIP_LINT_TASK: "1"
+   SKIP_LINT_TASK: "true"
+   SKIP_LINT_TASK: "false"
+   SKIP_LINT_TASK: "skip"
+   SKIP_LINT_TASK: "unicorn"
 
-The `cirrus-ci`_ tests are run automatically against the `Iris`_ master
-repository when a pull request is submitted, updated or merged.
 
 GitHub Checklist
 ****************
@@ -50,6 +86,10 @@ passing:
 
 .. image:: ci_checks.png
 
-If any CI checks fail, then the pull request is unlikely to be merged to the
+If any CI tasks fail, then the pull request is unlikely to be merged to the
 Iris target branch by a core developer.
 
+
+.. _Cirrus-CI Dashboard: https://cirrus-ci.com/github/SciTools/iris
+.. _Cirrus-CI Documentation: https://cirrus-ci.org/guide/writing-tasks/
+

docs/src/developers_guide/contributing_code_formatting.rst

@@ -58,6 +58,4 @@ will look similar to::
           their officially documentation for more information.
 
 
-.. _black: https://black.readthedocs.io/en/stable/
-.. _flake8: https://flake8.pycqa.org/en/stable/
 .. _pre-commit: https://pre-commit.com/