diff --git a/.cirrus.yml b/.cirrus.yml index 52b05d5f..856149ff 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -11,8 +11,13 @@ # container: image: python:3.8 + cpu: 2 + memory: 4G env: + # Skip specific tasks by name. Set to a non-empty string to skip. + SKIP_LINT_TASK: "" + SKIP_TEST_TASK: "" # Maximum cache period (in weeks) before forcing a new cache upload. CACHE_PERIOD: "2" # Increment the build number to force new conda cache upload. @@ -21,50 +26,38 @@ env: NOX_CACHE_BUILD: "0" # Increment the build number to force new pip cache upload. PIP_CACHE_BUILD: "0" - # Pip package to be upgraded/installed. - PIP_CACHE_PACKAGES: "pip setuptools wheel nox" + # Pip package to be installed. + PIP_CACHE_PACKAGES: "pip setuptools wheel nox pyyaml" + # Conda packages to be installed. + CONDA_CACHE_PACKAGES: "nox pip pyyaml" # # Linting # lint_task: + only_if: ${SKIP_LINT_TASK} == "" auto_cancellation: true - name: "Lint: flake8" + name: "${CIRRUS_OS}: flake8 and black" pip_cache: folder: ~/.cache/pip fingerprint_script: - - echo "${CIRRUS_OS} py${PYTHON_VERSION}" - - echo "$(date +%Y).$(expr $(date +%U) / ${CACHE_PERIOD}):${PIP_CACHE_BUILD} ${PIP_CACHE_PACKAGES}" + - echo "${CIRRUS_TASK_NAME} py${PYTHON_VERSION}" + - echo "${PIP_CACHE_PACKAGES}" + - echo "$(date +%Y).$(expr $(date +%U) / ${CACHE_PERIOD}):${PIP_CACHE_BUILD}" lint_script: - pip list - python -m pip install --retries 3 --upgrade ${PIP_CACHE_PACKAGES} - pip list - - nox --session lint - - -# -# Formatting -# -style_task: - auto_cancellation: true - name: "Format: black" - pip_cache: - folder: ~/.cache/pip - fingerprint_script: - - echo "${CIRRUS_OS} py${PYTHON_VERSION}" - - echo "$(date +%Y).$(expr $(date +%U) / ${CACHE_PERIOD}):${PIP_CACHE_BUILD} ${PIP_CACHE_PACKAGES}" - style_script: - - pip list - - python -m pip install --retries 3 --upgrade ${PIP_CACHE_PACKAGES} - - pip list - - nox --session style + - nox --session flake8 + - nox --session black # # Testing (Linux) # -linux_task: +test_task: + only_if: ${SKIP_TEST_TASK} == "" auto_cancellation: true matrix: env: @@ -73,20 +66,18 @@ linux_task: PY_VER: "3.7" env: PY_VER: "3.8" - COVERAGE: "pytest-cov codecov" - name: "Linux: py${PY_VER}" + COVERAGE: "true" + name: "${CIRRUS_OS}: py${PY_VER} tests" container: image: gcc:latest env: PATH: ${HOME}/miniconda/bin:${PATH} - CODECOV_TOKEN: "ENCRYPTED\ - [1ed538b97a8d005bdd5ab729de009ac38a2b53389edb0912\ - d2e76f5ce1e71c5f7bdea80a79492b57af54691c8936bdc7]" conda_cache: folder: ${HOME}/miniconda fingerprint_script: - wget --quiet https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh - echo "${CIRRUS_OS} $(sha256sum miniconda.sh)" + - echo "${CONDA_CACHE_PACKAGES}" - echo "$(date +%Y).$(expr $(date +%U) / ${CACHE_PERIOD}):${CONDA_CACHE_BUILD}" populate_script: - bash miniconda.sh -b -p ${HOME}/miniconda @@ -94,84 +85,13 @@ linux_task: - conda config --set show_channel_urls True - conda config --add channels conda-forge - conda update --quiet --name base conda - - conda install --quiet --name base nox pip - check_script: - - conda info --all - - conda list --name base - - conda list --name base --explicit - nox_cache: - folder: ${CIRRUS_WORKING_DIR}/.nox - fingerprint_script: - - echo "${CIRRUS_OS} tests ${PY_VER}" - - echo "$(date +%Y).$(expr $(date +%U) / ${CACHE_PERIOD}):${NOX_CACHE_BUILD}" - - cat ${CIRRUS_WORKING_DIR}/requirements/py$(echo ${PY_VER} | tr -d ".").yml - - if [ -n "${COVERAGE}" ]; then echo "${COVERAGE}"; fi - test_script: - - nox --session tests - - -# -# Testing (macOS) -# -osx_task: - auto_cancellation: true - matrix: - env: - PY_VER: "3.8" - name: "OSX: py${PY_VER}" - osx_instance: - image: catalina-xcode - env: - PATH: ${HOME}/miniconda/bin:${PATH} - conda_cache: - folder: ${HOME}/miniconda - fingerprint_script: - - wget --quiet https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh -O miniconda.sh - - echo "${CIRRUS_OS} $(shasum -a 256 miniconda.sh)" - - echo "$(date +%Y).$(expr $(date +%U) / ${CACHE_PERIOD}):${CONDA_CACHE_BUILD}" - populate_script: - - bash miniconda.sh -b -p ${HOME}/miniconda - - conda config --set always_yes yes --set changeps1 no - - conda config --set show_channel_urls True - - conda config --add channels conda-forge - - conda update --quiet --name base conda - - conda install --quiet --name base nox pip - check_script: - - conda info --all - - conda list --name base - - conda list --name base --explicit + - conda install --quiet --name base ${CONDA_CACHE_PACKAGES} nox_cache: folder: ${CIRRUS_WORKING_DIR}/.nox fingerprint_script: - - echo "${CIRRUS_OS} tests ${PY_VER}" + - echo "${CIRRUS_TASK_NAME}" - echo "$(date +%Y).$(expr $(date +%U) / ${CACHE_PERIOD}):${NOX_CACHE_BUILD}" - - cat ${CIRRUS_WORKING_DIR}/requirements/py$(echo ${PY_VER} | tr -d ".").yml + - sha256sum ${CIRRUS_WORKING_DIR}/requirements/py$(echo ${PY_VER} | tr -d ".").yml + - if [ -n "${IRIS_SOURCE}" ]; then echo "${IRIS_SOURCE}"; fi test_script: - - nox --session tests - - -# -# esmpy is unavailable from conda-forge for win -# -# windows_task: -# auto_cancellation: true -# matrix: -# env: -# PY_VER: "3.8" -# name: "Windows: py${PY_VER}" -# windows_container: -# image: cirrusci/windowsservercore:2019 -# env: -# ANACONDA_LOCATION: $USERPROFILE\anaconda -# PATH: $ANACONDA_LOCATION\Scripts;$ANACONDA_LOCATION;$PATH -# PYTHON_ARCH: 64 -# install_script: -# - choco install -y openssl.light -# - choco install -y miniconda3 --params="'/D:%ANACONDA_LOCATION%'" -# conda_script: -# - conda config --set always_yes yes --set changeps1 no -# - conda config --set show_channel_urls True -# - conda config --add channels conda-forge -# - conda install --quiet --name base nox pip -# test_script: -# - nox --session tests + - nox --session tests -- --verbose diff --git a/.gitignore b/.gitignore index b6e47617..428aa168 100644 --- a/.gitignore +++ b/.gitignore @@ -127,3 +127,6 @@ dmypy.json # Pyre type checker .pyre/ + +# Editors and IDEs +.idea diff --git a/noxfile.py b/noxfile.py index ebc27076..405ffa77 100644 --- a/noxfile.py +++ b/noxfile.py @@ -5,10 +5,12 @@ """ +import hashlib import os from pathlib import Path import nox +import yaml #: Default to reusing any pre-existing nox environments. @@ -18,16 +20,150 @@ PACKAGE = "esmf_regrid" #: Cirrus-CI environment variable hook. -PY_VER = os.environ.get("PY_VER", "3.8") +PY_VER = os.environ.get("PY_VER", ["3.6", "3.7", "3.8"]) #: Cirrus-CI environment variable hook. COVERAGE = os.environ.get("COVERAGE", False) +#: Cirrus-CI environment variable hook. +IRIS_SOURCE = os.environ.get("IRIS_SOURCE", None) + +COVERAGE_PACKAGES = ["pytest-cov", "codecov"] +IRIS_GITHUB = "https://github.com/scitools/iris.git" + + +def _cache_venv(session, fname): + """ + Cache the nox session environment. + + This consists of saving a hexdigest (sha256) of the associated + conda requirements YAML file. + + Parameters + ---------- + session: object + A `nox.sessions.Session` object. + fname: str + Requirements filename that defines this environment. + + """ + python_version = session.python.replace(".", "") + with open(fname, "rb") as fi: + hexdigest = hashlib.sha256(fi.read()).hexdigest() + tmp_dir = Path(session.create_tmp()) + cache = tmp_dir / f"py{python_version}.sha" + with open(cache, "w") as fo: + fo.write(hexdigest) + + +def _combine_requirements(primary, secondary, ignore=None): + """ + Combine the conda environment YAML files together into one. + + Parameters + ---------- + primary: str + The filename of the primary YAML conda environment. + secondary: str + The filename of the subordinate YAML conda environment. + ignore: str, optional + The prefix of any package name to be ignored from the + combined dependency requirements. + + Returns + ------- + dict + A dictionary of the combined YAML conda environments. + + """ + with open(primary, "r") as fi: + result = yaml.load(fi, Loader=yaml.FullLoader) + with open(secondary, "r") as fi: + secondary = yaml.load(fi, Loader=yaml.FullLoader) + # Combine the channels and dependencies only. + for key in ["channels", "dependencies"]: + result[key] = sorted(set(result[key]).union(secondary[key])) + if ignore: + # Filter out any specific prefixed package dependencies. + result["dependencies"] = [ + spec for spec in result["dependencies"] if not spec.startswith(ignore) + ] + return result + + +def _get_iris_github_artifact(session): + """ + Determine whether an Iris source artifact from GitHub is required. + + This can be an Iris branch name, commit sha or tag name. + + Parameters + ---------- + session: object + A `nox.sessions.Session` object. + + Returns + ------- + str + The Iris GitHub artifact. + + """ + result = IRIS_SOURCE + # The CLI overrides the environment variable. + for arg in session.posargs: + if arg.startswith("--iris="): + parts = arg.split("=") + if len(parts) == 2: + result = parts[1].strip() + break + if result: + parts = result.strip().split(":") + result = None + if len(parts) == 2: + repo, artifact = parts + if repo.startswith("'") or repo.startswith('"'): + repo = repo[1:] + if repo.lower() == "github": + result = artifact + if result.endswith("'") or result.endswith('"'): + result = result[:-1] + return result + + +def _venv_cached(session, fname): + """ + Determine whether the nox session environment has been cached. + + Parameters + ---------- + session: object + A `nox.sessions.Session` object. + fname: str + Requirements filename that defines this environment. + + Returns + ------- + bool + Whether the session has been cached. + + """ + result = False + tmp_dir = Path(session.create_tmp()) + python_version = session.python.replace(".", "") + cache = tmp_dir / f"py{python_version}.sha" + if cache.is_file(): + with open(fname, "rb") as fi: + hexdigest = hashlib.sha256(fi.read()).hexdigest() + with open(cache, "r") as fi: + cached = fi.read() + result = cached == hexdigest + return result + @nox.session -def lint(session): +def flake8(session): """ - Perform linting of the code-base. + Perform flake8 linting of the code-base. Parameters ---------- @@ -44,9 +180,9 @@ def lint(session): @nox.session -def style(session): +def black(session): """ - Perform format checking of the code-base. + Perform black format checking of the code-base. Parameters ---------- @@ -62,10 +198,10 @@ def style(session): session.run("black", "--check", __file__) -@nox.session(python=[PY_VER], venv_backend="conda") +@nox.session(python=PY_VER, venv_backend="conda") def tests(session): """ - Support for conda in nox is relatively new and maturing. + Perform esmf-regrid integration and unit tests. Parameters ---------- @@ -79,26 +215,67 @@ def tests(session): - https://github.com/theacodes/nox/issues/260 """ - # Determine whether pytest is installed as part of - # our conda requirements. - pytest = Path(session.bin) / "pytest" - if not pytest.is_file(): - # Determine the conda requirements yaml file. - fname = f"requirements/py{PY_VER.replace('.', '')}.yml" + artifact = _get_iris_github_artifact(session) + python_version = session.python.replace(".", "") + requirements_fname = f"requirements/py{python_version}.yml" + + if artifact: + tmp_dir = Path(session.create_tmp()) + artifact_dir = tmp_dir / "iris" + cwd = Path.cwd() + if not artifact_dir.is_dir(): + session.run("git", "clone", IRIS_GITHUB, str(artifact_dir), external=True) + session.cd(artifact_dir) + session.run("git", "fetch", "origin", external=True) + session.run("git", "checkout", artifact, external=True) + session.cd(str(cwd)) + iris_requirements_fname = ( + f"{artifact_dir}/requirements/ci/py{python_version}.yml" + ) + requirements_yaml = _combine_requirements( + requirements_fname, iris_requirements_fname, ignore="iris" + ) + requirements_fname = tmp_dir / "requirements.yml" + with open(requirements_fname, "w") as fo: + yaml.dump(requirements_yaml, fo) + + # Install the package requirements. + if not _venv_cached(session, requirements_fname): # Back-door approach to force nox to use "conda env update". command = ( "conda", "env", "update", f"--prefix={session.virtualenv.location}", - f"--file={fname}", + f"--file={requirements_fname}", "--prune", ) session._run(*command, silent=True, external="error") + _cache_venv(session, requirements_fname) + + if artifact: + # Install the iris source in develop mode. + session.install("--no-deps", "--editable", str(artifact_dir)) + + # Install the esmf-regrid source in develop mode. + session.install("--no-deps", "--editable", ".") + + # Determine whether verbose diagnostics have been requested from the command line. + verbose = "-v" in session.posargs or "--verbose" in session.posargs + + if verbose: + session.run("conda", "info") + session.run("conda", "list", f"--prefix={session.virtualenv.location}") + session.run( + "conda", + "list", + f"--prefix={session.virtualenv.location}", + "--explicit", + ) if COVERAGE: # Execute the tests with code coverage. - session.conda_install("--channel=conda-forge", *COVERAGE.split()) + session.conda_install("--channel=conda-forge", *COVERAGE_PACKAGES) session.run("pytest", "--cov-report=xml", "--cov") session.run("codecov") else: diff --git a/requirements/py36.yml b/requirements/py36.yml index d1c046e3..5852ab89 100644 --- a/requirements/py36.yml +++ b/requirements/py36.yml @@ -20,6 +20,5 @@ dependencies: - flake8 - flake8-docstrings - flake8-import-order - - nox - pre-commit - pytest diff --git a/requirements/py37.yml b/requirements/py37.yml index cb3808a7..b27bea38 100644 --- a/requirements/py37.yml +++ b/requirements/py37.yml @@ -20,6 +20,5 @@ dependencies: - flake8 - flake8-docstrings - flake8-import-order - - nox - pre-commit - pytest diff --git a/requirements/py38.yml b/requirements/py38.yml index 4581da93..d3cdcaf3 100644 --- a/requirements/py38.yml +++ b/requirements/py38.yml @@ -20,6 +20,5 @@ dependencies: - flake8 - flake8-docstrings - flake8-import-order - - nox - pre-commit - pytest