From 3143f1af44f96088eb417f34cc111d7361b4c6d8 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Wed, 14 Jul 2021 07:39:00 +0200 Subject: [PATCH] Move CI-integration images to ghcr.io (#16797) This is the final step of moving the images used for CI integration to `ghcr.io` from DockerHub. With Publicly available images with self-management provided by GitHub, we can finally move to keep the images "properly" - i.e. each image is separate and tag is only image version. Part of #16555 --- BREEZE.rst | 2 +- CI.rst | 6 +++--- breeze | 2 +- kubernetes_tests/test_kubernetes_pod_operator.py | 2 +- scripts/ci/docker-compose/integration-kerberos.yml | 2 +- scripts/ci/docker-compose/integration-openldap.yml | 2 +- scripts/ci/docker-compose/integration-trino.yml | 2 +- scripts/ci/dockerfiles/apache-rat/build_and_push.sh | 11 +++++------ scripts/ci/dockerfiles/bats/build_and_push.sh | 11 +++++------ .../dockerfiles/krb5-kdc-server/build_and_push.sh | 11 +++++------ scripts/ci/dockerfiles/stress/build_and_push.sh | 12 ++++++------ scripts/ci/dockerfiles/trino/Dockerfile | 2 +- scripts/ci/dockerfiles/trino/build_and_push.sh | 13 ++++++------- scripts/ci/static_checks/bats_tests.sh | 7 +++---- scripts/ci/static_checks/check_license.sh | 2 +- tests/kubernetes/pod.yaml | 2 +- tests/kubernetes/test_pod_generator.py | 4 ++-- 17 files changed, 44 insertions(+), 49 deletions(-) diff --git a/BREEZE.rst b/BREEZE.rst index eef7aafb9a66..7ae9b8b8ef79 100644 --- a/BREEZE.rst +++ b/BREEZE.rst @@ -1200,7 +1200,7 @@ This is the current syntax for `./breeze <./breeze>`_: you want to pull the image with specific COMMIT_SHA tag. 'breeze shell \ - -- github-image-id 9a621eaa394c0a0a336f8e1b31b35eff4e4ee86e' - pull/use image with SHA + --github-image-id 9a621eaa394c0a0a336f8e1b31b35eff4e4ee86e' - pull/use image with SHA 'breeze \ --github-image-id 9a621eaa394c0a0a336f8e1b31b35eff4e4ee86e' - pull/use image with SHA diff --git a/CI.rst b/CI.rst index d0eec1bdb34f..fedd3007b7cb 100644 --- a/CI.rst +++ b/CI.rst @@ -78,7 +78,7 @@ this image can be built only once and used by all the jobs running tests. The im rather than build it from the scratch. Pulling such image takes ~ 1 minute, thanks to that we are saving a lot of precious time for jobs. -We use `GitHub Container Registry `_ +We use `GitHub Container Registry `_. ``GITHUB_TOKEN`` is needed to push to the registry and we configured scopes of the tokens in our jobs to be able to write to the registry. @@ -411,9 +411,9 @@ The following components are part of the CI infrastructure * **GitHub Actions** - (GA) UI + execution engine for our jobs * **GA CRON trigger** - GitHub Actions CRON triggering our jobs * **GA Workers** - virtual machines running our jobs at GitHub Actions (max 20 in parallel) -* **GitHub Image Registry**- image registry used as build cache for CI jobs. +* **GitHub Image Registry** - image registry used as build cache for CI jobs. It is at https://ghcr.io/apache/airflow -* **DockerHub Image Registry**- image registry used to pull base Python images and (manually) publish +* **DockerHub Image Registry** - image registry used to pull base Python images and (manually) publish the released Production Airflow images. It is at https://dockerhub.com/apache/airflow * **Official Images** (future) - these are official images that are prominently visible in DockerHub. We aim our images to become official images so that you will be able to pull them diff --git a/breeze b/breeze index 9ceecf8f35ee..b026ebb30547 100755 --- a/breeze +++ b/breeze @@ -1688,7 +1688,7 @@ ${CMDNAME} shell [FLAGS] [-- ] you want to pull the image with specific COMMIT_SHA tag. '${CMDNAME} shell \\ - -- github-image-id 9a621eaa394c0a0a336f8e1b31b35eff4e4ee86e' - pull/use image with SHA + --github-image-id 9a621eaa394c0a0a336f8e1b31b35eff4e4ee86e' - pull/use image with SHA '${CMDNAME} \\ --github-image-id 9a621eaa394c0a0a336f8e1b31b35eff4e4ee86e' - pull/use image with SHA diff --git a/kubernetes_tests/test_kubernetes_pod_operator.py b/kubernetes_tests/test_kubernetes_pod_operator.py index 93d0a4936237..604e5f2da33e 100644 --- a/kubernetes_tests/test_kubernetes_pod_operator.py +++ b/kubernetes_tests/test_kubernetes_pod_operator.py @@ -867,7 +867,7 @@ def test_pod_template_file(self, mock_client, monitor_mock, start_mock): 'command': ['stress'], 'env': [], 'envFrom': [], - 'image': 'apache/airflow-ci:stress-2021.04.28-1.0.4', + 'image': 'ghcr.io/apache/airflow-stress:1.0.4-2021.07.04', 'name': 'base', 'ports': [], 'resources': {'limits': {'memory': '200Mi'}, 'requests': {'memory': '100Mi'}}, diff --git a/scripts/ci/docker-compose/integration-kerberos.yml b/scripts/ci/docker-compose/integration-kerberos.yml index b9ca4cc6ccf1..0e79c6aca58b 100644 --- a/scripts/ci/docker-compose/integration-kerberos.yml +++ b/scripts/ci/docker-compose/integration-kerberos.yml @@ -18,7 +18,7 @@ version: "2.2" services: kdc-server-example-com: - image: apache/airflow-ci:krb5-kdc-server-2021.04.28 + image: ghcr.io/apache/airflow-krb5-kdc-server:2021.07.04 hostname: krb5-kdc-server-example-com domainname: example.com networks: diff --git a/scripts/ci/docker-compose/integration-openldap.yml b/scripts/ci/docker-compose/integration-openldap.yml index 60ff58a30124..eea32460bae9 100644 --- a/scripts/ci/docker-compose/integration-openldap.yml +++ b/scripts/ci/docker-compose/integration-openldap.yml @@ -18,7 +18,7 @@ version: "2.2" services: openldap: - image: apache/airflow-ci:openldap-2020.07.10-2.4.50 + image: ghcr.io/apache/airflow-openldap:2.4.50-2021.07.04 command: "--copy-service" environment: - LDAP_DOMAIN=example.com diff --git a/scripts/ci/docker-compose/integration-trino.yml b/scripts/ci/docker-compose/integration-trino.yml index 822a2ff045f3..e381ced8aba9 100644 --- a/scripts/ci/docker-compose/integration-trino.yml +++ b/scripts/ci/docker-compose/integration-trino.yml @@ -18,7 +18,7 @@ version: "2.2" services: trino: - image: apache/airflow-ci:trino-2021.04.28 + image: ghcr.io/apache/airflow-trino:359-2021.07.04 container_name: trino hostname: trino domainname: example.com diff --git a/scripts/ci/dockerfiles/apache-rat/build_and_push.sh b/scripts/ci/dockerfiles/apache-rat/build_and_push.sh index ceed53469198..228b30145662 100755 --- a/scripts/ci/dockerfiles/apache-rat/build_and_push.sh +++ b/scripts/ci/dockerfiles/apache-rat/build_and_push.sh @@ -16,15 +16,13 @@ # specific language governing permissions and limitations # under the License. set -euo pipefail -DOCKERHUB_USER=${DOCKERHUB_USER:="apache"} -DOCKERHUB_REPO=${DOCKERHUB_REPO:="airflow-ci"} -readonly DOCKERHUB_USER -readonly DOCKERHUB_REPO +GITHUB_REPOSITORY=${GITHUB_REPOSITORY:="apache/airflow"} +readonly GITHUB_REPOSITORY APACHERAT_VERSION="0.13" readonly APACHERAT_VERSION -AIRFLOW_APACHERAT_VERSION="2021.04.28" +AIRFLOW_APACHERAT_VERSION="2021.07.04" readonly AIRFLOW_APACHERAT_VERSION COMMIT_SHA=$(git rev-parse HEAD) @@ -32,7 +30,7 @@ readonly COMMIT_SHA cd "$( dirname "${BASH_SOURCE[0]}" )" || exit 1 -TAG="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:apache-rat-${AIRFLOW_APACHERAT_VERSION}-${APACHERAT_VERSION}" +TAG="ghcr.io/${GITHUB_REPOSITORY}-apache-rat:${APACHERAT_VERSION}-${AIRFLOW_APACHERAT_VERSION}" readonly TAG docker build . \ @@ -40,6 +38,7 @@ docker build . \ --build-arg "APACHERAT_VERSION=${APACHERAT_VERSION}" \ --build-arg "AIRFLOW_APACHERAT_VERSION=${AIRFLOW_APACHERAT_VERSION}" \ --build-arg "COMMIT_SHA=${COMMIT_SHA}" \ + --label "org.opencontainers.image.source=https://github.com/${GITHUB_REPOSITORY}" \ --tag "${TAG}" docker push "${TAG}" diff --git a/scripts/ci/dockerfiles/bats/build_and_push.sh b/scripts/ci/dockerfiles/bats/build_and_push.sh index 1f35c9a92845..20b6c02fda7a 100755 --- a/scripts/ci/dockerfiles/bats/build_and_push.sh +++ b/scripts/ci/dockerfiles/bats/build_and_push.sh @@ -16,10 +16,8 @@ # specific language governing permissions and limitations # under the License. set -euo pipefail -DOCKERHUB_USER=${DOCKERHUB_USER:="apache"} -DOCKERHUB_REPO=${DOCKERHUB_REPO:="airflow-ci"} -readonly DOCKERHUB_USER -readonly DOCKERHUB_REPO +GITHUB_REPOSITORY=${GITHUB_REPOSITORY:="apache/airflow"} +readonly GITHUB_REPOSITORY BATS_VERSION="1.2.1" BATS_ASSERT_VERSION="2.0.0" @@ -30,7 +28,7 @@ readonly BATS_ASSERT_VERSION readonly BATS_SUPPORT_VERSION readonly BATS_FILE_VERSION -AIRFLOW_BATS_VERSION="2021.04.28" +AIRFLOW_BATS_VERSION="2021.07.04" readonly AIRFLOW_BATS_VERSION COMMIT_SHA=$(git rev-parse HEAD) @@ -38,7 +36,7 @@ readonly COMMIT_SHA cd "$( dirname "${BASH_SOURCE[0]}" )" || exit 1 -TAG="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:bats-${AIRFLOW_BATS_VERSION}-${BATS_VERSION}" +TAG="ghcr.io/${GITHUB_REPOSITORY}-bats:${BATS_VERSION}-${AIRFLOW_BATS_VERSION}" readonly TAG docker build . \ @@ -48,6 +46,7 @@ docker build . \ --build-arg "BATS_FILE_VERSION=${BATS_FILE_VERSION}" \ --build-arg "BATS_ASSERT_VERSION=${BATS_ASSERT_VERSION}" \ --build-arg "COMMIT_SHA=${COMMIT_SHA}" \ + --label "org.opencontainers.image.source=https://github.com/${GITHUB_REPOSITORY}" \ --tag "${TAG}" docker push "${TAG}" diff --git a/scripts/ci/dockerfiles/krb5-kdc-server/build_and_push.sh b/scripts/ci/dockerfiles/krb5-kdc-server/build_and_push.sh index e1a242aeca41..6ad4354e2aec 100755 --- a/scripts/ci/dockerfiles/krb5-kdc-server/build_and_push.sh +++ b/scripts/ci/dockerfiles/krb5-kdc-server/build_and_push.sh @@ -16,12 +16,10 @@ # specific language governing permissions and limitations # under the License. set -euo pipefail -DOCKERHUB_USER=${DOCKERHUB_USER:="apache"} -DOCKERHUB_REPO=${DOCKERHUB_REPO:="airflow-ci"} -readonly DOCKERHUB_USER -readonly DOCKERHUB_REPO +GITHUB_REPOSITORY=${GITHUB_REPOSITORY:="apache/airflow"} +readonly GITHUB_REPOSITORY -AIRFLOW_KRB5KDCSERVER_VERSION="2021.04.28" +AIRFLOW_KRB5KDCSERVER_VERSION="2021.07.04" readonly AIRFLOW_KRB5KDCSERVER_VERSION COMMIT_SHA=$(git rev-parse HEAD) @@ -29,13 +27,14 @@ readonly COMMIT_SHA cd "$( dirname "${BASH_SOURCE[0]}" )" || exit 1 -TAG="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:krb5-kdc-server-${AIRFLOW_KRB5KDCSERVER_VERSION}" +TAG="ghcr.io/${GITHUB_REPOSITORY}-krb5-kdc-server:${AIRFLOW_KRB5KDCSERVER_VERSION}" readonly TAG docker build . \ --pull \ --build-arg "AIRFLOW_KRB5KDCSERVER_VERSION=${AIRFLOW_KRB5KDCSERVER_VERSION}" \ --build-arg "COMMIT_SHA=${COMMIT_SHA}" \ + --label "org.opencontainers.image.source=https://github.com/${GITHUB_REPOSITORY}" \ --tag "${TAG}" docker push "${TAG}" diff --git a/scripts/ci/dockerfiles/stress/build_and_push.sh b/scripts/ci/dockerfiles/stress/build_and_push.sh index c1584564c167..ea3144d0efce 100755 --- a/scripts/ci/dockerfiles/stress/build_and_push.sh +++ b/scripts/ci/dockerfiles/stress/build_and_push.sh @@ -16,14 +16,13 @@ # specific language governing permissions and limitations # under the License. set -euo pipefail -DOCKERHUB_USER=${DOCKERHUB_USER:="apache"} -DOCKERHUB_REPO=${DOCKERHUB_REPO:="airflow-ci"} -readonly DOCKERHUB_USER -readonly DOCKERHUB_REPO +GITHUB_REPOSITORY=${GITHUB_REPOSITORY:="apache/airflow"} +readonly GITHUB_REPOSITORY + STRESS_VERSION="1.0.4" readonly STRESS_VERSION -AIRFLOW_STRESS_VERSION="2021.04.28" +AIRFLOW_STRESS_VERSION="2021.07.04" readonly AIRFLOW_STRESS_VERSION COMMIT_SHA=$(git rev-parse HEAD) @@ -31,7 +30,7 @@ readonly COMMIT_SHA cd "$( dirname "${BASH_SOURCE[0]}" )" || exit 1 -TAG="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:stress-${AIRFLOW_STRESS_VERSION}-${STRESS_VERSION}" +TAG="ghcr.io/${GITHUB_REPOSITORY}-stress:${STRESS_VERSION}-${AIRFLOW_STRESS_VERSION}" readonly TAG docker build . \ @@ -39,6 +38,7 @@ docker build . \ --build-arg "STRESS_VERSION=${STRESS_VERSION}" \ --build-arg "AIRFLOW_STRESS_VERSION=${AIRFLOW_STRESS_VERSION}" \ --build-arg "COMMIT_SHA=${COMMIT_SHA}" \ + --label "org.opencontainers.image.source=https://github.com/${GITHUB_REPOSITORY}" \ --tag "${TAG}" docker push "${TAG}" diff --git a/scripts/ci/dockerfiles/trino/Dockerfile b/scripts/ci/dockerfiles/trino/Dockerfile index 32ef0fd735fb..a36a8cc53e54 100644 --- a/scripts/ci/dockerfiles/trino/Dockerfile +++ b/scripts/ci/dockerfiles/trino/Dockerfile @@ -14,7 +14,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -ARG TRINO_VERSION="354" +ARG TRINO_VERSION="359" FROM trinodb/trino:${TRINO_VERSION} # Obtain root privileges diff --git a/scripts/ci/dockerfiles/trino/build_and_push.sh b/scripts/ci/dockerfiles/trino/build_and_push.sh index c5ebbc7c4a71..3f876f736b76 100755 --- a/scripts/ci/dockerfiles/trino/build_and_push.sh +++ b/scripts/ci/dockerfiles/trino/build_and_push.sh @@ -16,15 +16,13 @@ # specific language governing permissions and limitations # under the License. set -euo pipefail -DOCKERHUB_USER=${DOCKERHUB_USER:="apache"} -DOCKERHUB_REPO=${DOCKERHUB_REPO:="airflow-ci"} -readonly DOCKERHUB_USER -readonly DOCKERHUB_REPO +GITHUB_REPOSITORY=${GITHUB_REPOSITORY:="apache/airflow"} +readonly GITHUB_REPOSITORY -TRINO_VERSION="354" +TRINO_VERSION="359" readonly TRINO_VERSION -AIRFLOW_TRINO_VERSION="2021.04.28" +AIRFLOW_TRINO_VERSION="2021.07.04" readonly AIRFLOW_TRINO_VERSION COMMIT_SHA=$(git rev-parse HEAD) @@ -32,7 +30,7 @@ readonly COMMIT_SHA cd "$( dirname "${BASH_SOURCE[0]}" )" || exit 1 -TAG="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:trino-${AIRFLOW_TRINO_VERSION}" +TAG="ghcr.io/${GITHUB_REPOSITORY}-trino:${TRINO_VERSION}-${AIRFLOW_TRINO_VERSION}" readonly TAG docker build . \ @@ -40,6 +38,7 @@ docker build . \ --build-arg "TRINO_VERSION=${TRINO_VERSION}" \ --build-arg "AIRFLOW_TRINO_VERSION=${AIRFLOW_TRINO_VERSION}" \ --build-arg "COMMIT_SHA=${COMMIT_SHA}" \ + --label "org.opencontainers.image.source=https://github.com/${GITHUB_REPOSITORY}" \ --tag "${TAG}" docker push "${TAG}" diff --git a/scripts/ci/static_checks/bats_tests.sh b/scripts/ci/static_checks/bats_tests.sh index 6130ad5d12d4..55a62f3bdd3b 100755 --- a/scripts/ci/static_checks/bats_tests.sh +++ b/scripts/ci/static_checks/bats_tests.sh @@ -50,19 +50,18 @@ function run_bats_tests() { fi done + local airflow_bats_image="ghcr.io/apache/airflow-bats:1.2.1-2021.07.04" # deduplicate FS=" " read -r -a bats_arguments <<< "$(tr ' ' '\n' <<< "${bats_arguments[@]}" | sort -u | tr '\n' ' ' )" if [[ ${#@} == "0" ]]; then # Run al tests - docker run --workdir /airflow -v "$(pwd):/airflow" --rm \ - apache/airflow-ci:bats-2021.04.28-1.2.1 --tap /airflow/tests/bats/ + docker run --workdir /airflow -v "$(pwd):/airflow" --rm "${airflow_bats_image}" --tap /airflow/tests/bats/ elif [[ ${#bats_arguments} == "0" ]]; then # Skip running anything if all filtered out true else # Run selected tests - docker run --workdir /airflow -v "$(pwd):/airflow" --rm \ - apache/airflow-ci:bats-2021.04.28-1.2.1 --tap "${bats_arguments[@]}" + docker run --workdir /airflow -v "$(pwd):/airflow" --rm "${airflow_bats_image}" --tap "${bats_arguments[@]}" fi } diff --git a/scripts/ci/static_checks/check_license.sh b/scripts/ci/static_checks/check_license.sh index ce6e6faa15fd..ef365deee765 100755 --- a/scripts/ci/static_checks/check_license.sh +++ b/scripts/ci/static_checks/check_license.sh @@ -34,7 +34,7 @@ function run_check_license() { if ! docker_v run -v "${AIRFLOW_SOURCES}:/opt/airflow" -t \ --user "$(id -ur):$(id -gr)" \ --rm --env-file "${AIRFLOW_SOURCES}/scripts/ci/docker-compose/_docker.env" \ - apache/airflow-ci:apache-rat-2021.04.28-0.13 \ + ghcr.io/apache/airflow-apache-rat:0.13-2021.07.04 \ --exclude-file /opt/airflow/.rat-excludes \ --d /opt/airflow | tee "${AIRFLOW_SOURCES}/logs/rat-results.txt" ; then echo diff --git a/tests/kubernetes/pod.yaml b/tests/kubernetes/pod.yaml index 5e028801a237..b69844470cfd 100644 --- a/tests/kubernetes/pod.yaml +++ b/tests/kubernetes/pod.yaml @@ -23,7 +23,7 @@ metadata: spec: containers: - name: memory-demo-ctr - image: apache/airflow-ci:stress-2021.04.28-1.0.4 + image: ghcr.io/apache/airflow-stress:1.0.4-2021.07.04 resources: limits: memory: "200Mi" diff --git a/tests/kubernetes/test_pod_generator.py b/tests/kubernetes/test_pod_generator.py index 3ca60c124c5f..b288075d95f1 100644 --- a/tests/kubernetes/test_pod_generator.py +++ b/tests/kubernetes/test_pod_generator.py @@ -50,7 +50,7 @@ def setUp(self): { 'args': ['--vm', '1', '--vm-bytes', '150M', '--vm-hang', '1'], 'command': ['stress'], - 'image': 'apache/airflow-ci:stress-2021.04.28-1.0.4', + 'image': 'ghcr.io/apache/airflow-stress:1.0.4-2021.07.04', 'name': 'memory-demo-ctr', 'resources': {'limits': {'memory': '200Mi'}, 'requests': {'memory': '100Mi'}}, } @@ -688,7 +688,7 @@ def test_deserialize_model_string(self): spec: containers: - name: memory-demo-ctr - image: apache/airflow-ci:stress-2021.04.28-1.0.4 + image: ghcr.io/apache/airflow-stress:1.0.4-2021.07.04 resources: limits: memory: "200Mi"