From 08fe0904375077e43afc03c5529b427cfcf9aa37 Mon Sep 17 00:00:00 2001 From: Chalmer Lowe Date: Fri, 14 Jul 2023 11:25:26 -0400 Subject: [PATCH] feat: migrating off of circle/ci (#638) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * saving state of noxfile prior to owlbot.py edits * minor clean up of leftover artifacts from various experiments * linting and black * adds noxfile.py to excludes and removes noxfile edits in owlbot * Adds new kokoro file in presubmit * removes circle/ci config file * Update noxfile.py * Update noxfile.py * Update noxfile.py * remove limit on printing only KOKORO env variables (for testing) * updates build.sh with conda environmental variables * updates build.sh with conda environmental variables redux * removes the ci folder and files related to circle ci * updates env variable to display all vars not just KOKORO * removes additional content related to circle ci * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * updates owlbot with build.sh edits * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * updates owlbot with build.sh minor tweak * revert build.sh * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * testing regex change * revert build.sh * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --------- Co-authored-by: Owl Bot --- .circleci/config.yml | 34 -------- .kokoro/build.sh | 11 ++- .kokoro/presubmit/conda_test.cfg | 7 ++ ci/config_auth.sh | 13 --- ci/requirements-3.7-0.24.2.conda | 17 ---- ci/requirements-3.9-1.3.4.conda | 14 ---- ci/run_conda.sh | 23 ------ ci/run_tests.sh | 15 ---- noxfile.py | 81 +++++++++++++++++++ owlbot.py | 133 ++++++------------------------- tests/system/conftest.py | 11 --- tests/system/test_auth.py | 2 +- 12 files changed, 123 insertions(+), 238 deletions(-) delete mode 100644 .circleci/config.yml create mode 100644 .kokoro/presubmit/conda_test.cfg delete mode 100755 ci/config_auth.sh delete mode 100644 ci/requirements-3.7-0.24.2.conda delete mode 100644 ci/requirements-3.9-1.3.4.conda delete mode 100755 ci/run_conda.sh delete mode 100755 ci/run_tests.sh diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index e008054c..00000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright (c) 2017 pandas-gbq Authors All rights reserved. -# Use of this source code is governed by a BSD-style -# license that can be found in the LICENSE file. - -version: 2 -jobs: - # Conda - "conda-3.7": - docker: - - image: mambaorg/micromamba - environment: - PYTHON: "3.7" - PANDAS: "0.24.2" - steps: - - checkout - - run: ci/config_auth.sh - - run: ci/run_conda.sh - "conda-3.9": - docker: - - image: mambaorg/micromamba - environment: - PYTHON: "3.9" - PANDAS: "1.3.4" - steps: - - checkout - - run: ci/config_auth.sh - - run: ci/run_conda.sh - -workflows: - version: 2 - build: - jobs: - - "conda-3.7" - - "conda-3.9" diff --git a/.kokoro/build.sh b/.kokoro/build.sh index d02a78db..204ea03c 100755 --- a/.kokoro/build.sh +++ b/.kokoro/build.sh @@ -23,9 +23,18 @@ cd "${PROJECT_ROOT}" # Disable buffering, so that the logs stream through. export PYTHONUNBUFFERED=1 +export CONDA_EXE=/root/conda/bin/conda +export CONDA_PREFIX=/root/conda +export CONDA_PROMPT_MODIFIER=(base) +export _CE_CONDA= +export CONDA_SHLVL=1 +export CONDA_PYTHON_EXE=/root/conda/bin/python +export CONDA_DEFAULT_ENV=base +export PATH=/root/conda/bin:/root/conda/condabin:${PATH} + # Debug: show build environment -env | grep KOKORO +env # Setup service account credentials. export GOOGLE_APPLICATION_CREDENTIALS=${KOKORO_GFILE_DIR}/service-account.json diff --git a/.kokoro/presubmit/conda_test.cfg b/.kokoro/presubmit/conda_test.cfg new file mode 100644 index 00000000..6e3943f3 --- /dev/null +++ b/.kokoro/presubmit/conda_test.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Only run this nox session. +env_vars: { + key: "NOX_SESSION" + value: "conda_test" +} diff --git a/ci/config_auth.sh b/ci/config_auth.sh deleted file mode 100755 index cde115c7..00000000 --- a/ci/config_auth.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -# Copyright (c) 2017 pandas-gbq Authors All rights reserved. -# Use of this source code is governed by a BSD-style -# license that can be found in the LICENSE file. - -set -e -# Don't set -x, because we don't want to leak keys. -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" - -# Write key to file if present. -if [ ! -z "$SERVICE_ACCOUNT_KEY" ] ; then - echo "$SERVICE_ACCOUNT_KEY" | base64 --decode > "$DIR"/service_account.json -fi diff --git a/ci/requirements-3.7-0.24.2.conda b/ci/requirements-3.7-0.24.2.conda deleted file mode 100644 index 2d61383e..00000000 --- a/ci/requirements-3.7-0.24.2.conda +++ /dev/null @@ -1,17 +0,0 @@ -codecov -coverage -db-dtypes -fastavro -flake8 -freezegun -numpy -google-api-core -google-auth -google-cloud-bigquery -google-cloud-bigquery-storage -pyarrow -pydata-google-auth -pytest -pytest-cov -requests-oauthlib -tqdm diff --git a/ci/requirements-3.9-1.3.4.conda b/ci/requirements-3.9-1.3.4.conda deleted file mode 100644 index 1411fe5b..00000000 --- a/ci/requirements-3.9-1.3.4.conda +++ /dev/null @@ -1,14 +0,0 @@ -codecov -coverage -db-dtypes -fastavro -flake8 -freezegun -google-cloud-bigquery -google-cloud-bigquery-storage -numpy -pyarrow -pydata-google-auth -pytest -pytest-cov -tqdm diff --git a/ci/run_conda.sh b/ci/run_conda.sh deleted file mode 100755 index 11b5b569..00000000 --- a/ci/run_conda.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash -# Copyright (c) 2017 pandas-gbq Authors All rights reserved. -# Use of this source code is governed by a BSD-style -# license that can be found in the LICENSE file. - -set -e -x -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" - -eval "$(micromamba shell hook --shell=bash)" -micromamba activate - -# Install dependencies using (micro)mamba -# https://github.com/mamba-org/micromamba-docker -REQ="ci/requirements-${PYTHON}-${PANDAS}" -micromamba install -q pandas=$PANDAS python=${PYTHON} -c conda-forge; -micromamba install -q --file "$REQ.conda" -c conda-forge; -micromamba list -micromamba info - -python setup.py develop --no-deps - -# Run the tests -$DIR/run_tests.sh diff --git a/ci/run_tests.sh b/ci/run_tests.sh deleted file mode 100755 index 8a1d7f91..00000000 --- a/ci/run_tests.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash -# Copyright (c) 2017 pandas-gbq Authors All rights reserved. -# Use of this source code is governed by a BSD-style -# license that can be found in the LICENSE file. - -set -e -x -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" - -if [ -f "$DIR/service_account.json" ]; then - export GOOGLE_APPLICATION_CREDENTIALS="$DIR/service_account.json" -fi - -# Install test requirements -pip install coverage pytest pytest-cov flake8 codecov google-cloud-testutils -pytest -v -m "not local_auth" --cov=pandas_gbq --cov-report xml:/tmp/pytest-cov.xml --cov-fail-under=0 tests diff --git a/noxfile.py b/noxfile.py index 7ccecaac..135ce432 100644 --- a/noxfile.py +++ b/noxfile.py @@ -31,6 +31,7 @@ DEFAULT_PYTHON_VERSION = "3.8" + UNIT_TEST_PYTHON_VERSIONS = ["3.7", "3.8", "3.9", "3.10", "3.11"] UNIT_TEST_STANDARD_DEPENDENCIES = [ "mock", @@ -51,6 +52,11 @@ "3.9": [], } +CONDA_TEST_PYTHON_VERSIONS = [ + UNIT_TEST_PYTHON_VERSIONS[0], + UNIT_TEST_PYTHON_VERSIONS[-1], +] + SYSTEM_TEST_PYTHON_VERSIONS = ["3.7", "3.8", "3.9", "3.10", "3.11"] SYSTEM_TEST_STANDARD_DEPENDENCIES = [ "mock", @@ -514,3 +520,78 @@ def prerelease_deps(session): system_test_folder_path, *session.posargs, ) + + +def install_conda_unittest_dependencies(session, standard_deps, conda_forge_packages): + """Installs packages from conda forge, pypi, and locally.""" + + # Install from conda-forge and default conda package repos. + session.conda_install(*conda_forge_packages, channel=["defaults", "conda-forge"]) + + # Install from pypi for packages not readily available on conda forge. + session.install( + *standard_deps, + ) + + # Install via pip from the local repo, avoid doing dependency resolution + # via pip, so that we don't override any conda resolved dependencies + session.install("-e", ".", "--no-deps") + + +@nox.session(python=CONDA_TEST_PYTHON_VERSIONS, venv_backend="mamba") +def conda_test(session): + """Run test suite in a conda virtual environment. + + Installs all test dependencies, then installs this package. + NOTE: Some of these libraries are not readily available on conda-forge + at this time and are thus installed using pip after the base install of + libraries from conda-forge. + + We decided that it was more important to prove a base ability to install + using conda than to complicate things with adding a whole nother + set of constraints just for a conda install, so this install does not + attempt to constrain packages (i.e. in a constraints-x.x.txt file) + manually. + """ + + standard_deps = ( + UNIT_TEST_STANDARD_DEPENDENCIES + + UNIT_TEST_DEPENDENCIES + + UNIT_TEST_EXTERNAL_DEPENDENCIES + ) + + conda_forge_packages = [ + "db-dtypes", + "google-api-core", + "google-auth", + "google-auth-oauthlib", + "google-cloud-bigquery", + "google-cloud-bigquery-storage", + "numpy", + "pandas", + "pyarrow", + "pydata-google-auth", + "tqdm", + "protobuf", + ] + + install_conda_unittest_dependencies(session, standard_deps, conda_forge_packages) + + # Provide a list of all installed packages (both from conda forge and pip) + # for troubleshooting purposes. + session.run("mamba", "list") + + # Tests are limited to unit tests only, at this time. + session.run( + "py.test", + "--quiet", + f"--junitxml=unit_{session.python}_sponge_log.xml", + "--cov=pandas_gbq", + "--cov=tests/unit", + "--cov-append", + "--cov-config=.coveragerc", + "--cov-report=", + "--cov-fail-under=0", + os.path.join("tests", "unit"), + *session.posargs, + ) diff --git a/owlbot.py b/owlbot.py index 9c9454f8..e648334d 100644 --- a/owlbot.py +++ b/owlbot.py @@ -52,10 +52,11 @@ excludes=[ # pandas-gbq was originally licensed BSD-3-Clause License "LICENSE", - # Mulit-processing note isn't relevant, as pandas_gbq is responsible for + # Multi-processing note isn't relevant, as pandas_gbq is responsible for # creating clients, not the end user. "docs/multiprocessing.rst", - "README.rst", + "noxfile.py", + "README.rst", ], ) @@ -64,123 +65,37 @@ # ---------------------------------------------------------------------------- s.replace( - ["noxfile.py"], - r"import pathlib\s+import shutil", - "import pathlib\nimport re\nimport shutil", -) - -s.replace( - ["noxfile.py"], r"[\"']google[\"']", '"pandas_gbq"', + [".github/header-checker-lint.yml"], '"Google LLC"', '"pandas-gbq Authors"', ) +# Work around bug in templates https://github.com/googleapis/synthtool/pull/1335 +s.replace(".github/workflows/unittest.yml", "--fail-under=100", "--fail-under=96") +# Add environment variables to build.sh to support conda virtualenv +# installations s.replace( - ["noxfile.py"], "--cov=google", "--cov=pandas_gbq", -) - -# Workaround for https://github.com/googleapis/synthtool/issues/1317 -s.replace( - ["noxfile.py"], r'extras = "\[\]"', 'extras = ""', + [".kokoro/build.sh"], + "export PYTHONUNBUFFERED=1", + r"""export PYTHONUNBUFFERED=1 +export CONDA_EXE=/root/conda/bin/conda +export CONDA_PREFIX=/root/conda +export CONDA_PROMPT_MODIFIER=(base) +export _CE_CONDA= +export CONDA_SHLVL=1 +export CONDA_PYTHON_EXE=/root/conda/bin/python +export CONDA_DEFAULT_ENV=base +export PATH=/root/conda/bin:/root/conda/condabin:${PATH} +""", ) -s.replace( - ["noxfile.py"], - r"@nox.session\(python=DEFAULT_PYTHON_VERSION\)\s+def cover\(session\):", - r"""@nox.session(python=DEFAULT_PYTHON_VERSION) -def prerelease(session): - session.install( - "--extra-index-url", - "https://pypi.fury.io/arrow-nightlies/", - "--prefer-binary", - "--pre", - "--upgrade", - "pyarrow", - ) - session.install( - "--extra-index-url", - "https://pypi.anaconda.org/scipy-wheels-nightly/simple", - "--prefer-binary", - "--pre", - "--upgrade", - "pandas", - ) - session.install( - "--prefer-binary", - "--pre", - "--upgrade", - "google-api-core", - "google-cloud-bigquery", - "google-cloud-bigquery-storage", - "google-cloud-core", - "google-resumable-media", - # Exclude version 1.49.0rc1 which has a known issue. See https://github.com/grpc/grpc/pull/30642 - "grpcio!=1.49.0rc1", - ) - session.install( - "freezegun", - "google-cloud-datacatalog", - "google-cloud-storage", - "google-cloud-testutils", - "IPython", - "mock", - "psutil", - "pytest", - "pytest-cov", - ) - - # Because we test minimum dependency versions on the minimum Python - # version, the first version we test with in the unit tests sessions has a - # constraints file containing all dependencies and extras. - with open( - CURRENT_DIRECTORY - / "testing" - / f"constraints-{UNIT_TEST_PYTHON_VERSIONS[0]}.txt", - encoding="utf-8", - ) as constraints_file: - constraints_text = constraints_file.read() - - # Ignore leading whitespace and comment lines. - deps = [ - match.group(1) - for match in re.finditer( - r"^\\s*(\\S+)(?===\\S+)", constraints_text, flags=re.MULTILINE - ) - ] - - # We use --no-deps to ensure that pre-release versions aren't overwritten - # by the version ranges in setup.py. - session.install(*deps) - session.install("--no-deps", "-e", ".[all]") - - # Print out prerelease package versions. - session.run("python", "-m", "pip", "freeze") - - # Run all tests, except a few samples tests which require extra dependencies. - session.run( - "py.test", - "--quiet", - f"--junitxml=prerelease_unit_{session.python}_sponge_log.xml", - os.path.join("tests", "unit"), - ) - session.run( - "py.test", - "--quiet", - f"--junitxml=prerelease_system_{session.python}_sponge_log.xml", - os.path.join("tests", "system"), - ) - - -@nox.session(python=DEFAULT_PYTHON_VERSION) -def cover(session):""", - re.MULTILINE, -) +# Enable display of all environmental variables, not just KOKORO related vars s.replace( - [".github/header-checker-lint.yml"], '"Google LLC"', '"pandas-gbq Authors"', + [".kokoro/build.sh"], + r"env \| grep KOKORO", + "env", ) -# Work around bug in templates https://github.com/googleapis/synthtool/pull/1335 -s.replace(".github/workflows/unittest.yml", "--fail-under=100", "--fail-under=96") # ---------------------------------------------------------------------------- # Samples templates diff --git a/tests/system/conftest.py b/tests/system/conftest.py index 4ba8bf31..9690446d 100644 --- a/tests/system/conftest.py +++ b/tests/system/conftest.py @@ -16,17 +16,6 @@ REPO_DIR = pathlib.Path(__file__).parent.parent.parent -# TODO: remove when fully migrated off of Circle CI -@pytest.fixture(scope="session", autouse=True) -def default_credentials(): - """Setup application default credentials for use in code samples.""" - # Written by the 'ci/config_auth.sh' script. - path = REPO_DIR / "ci" / "service_account.json" - - if path.is_file() and "GOOGLE_APPLICATION_CREDENTIALS" not in os.environ: - os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = str(path) - - @pytest.fixture(scope="session", autouse=True) def cleanup_datasets(bigquery_client: bigquery.Client): for dataset in bigquery_client.list_datasets(): diff --git a/tests/system/test_auth.py b/tests/system/test_auth.py index d9f7d096..ecedd973 100644 --- a/tests/system/test_auth.py +++ b/tests/system/test_auth.py @@ -12,7 +12,7 @@ from pandas_gbq import auth -IS_RUNNING_ON_CI = "CIRCLE_BUILD_NUM" in os.environ or "KOKORO_BUILD_ID" in os.environ +IS_RUNNING_ON_CI = "KOKORO_BUILD_ID" in os.environ def mock_default_credentials(scopes=None, request=None):