Move CI-integration images to ghcr.io (#16797)

This is the final step of moving the images used for CI integration to `ghcr.io` from DockerHub. With Publicly available images with self-management provided by GitHub, we can finally move to keep the images "properly" - i.e. each image is separate and tag is only image version. Part of #16555
apache · Jul 14, 2021 · 3143f1a · 3143f1a
1 parent fbfb08a
commit 3143f1a
Show file tree

Hide file tree

Showing 17 changed files with 44 additions and 49 deletions.
diff --git a/BREEZE.rst b/BREEZE.rst
@@ -1200,7 +1200,7 @@ This is the current syntax for  `./breeze <./breeze>`_:
         you want to pull the image with specific COMMIT_SHA tag.
 
         'breeze shell \
-              -- github-image-id 9a621eaa394c0a0a336f8e1b31b35eff4e4ee86e' - pull/use image with SHA
+              --github-image-id 9a621eaa394c0a0a336f8e1b31b35eff4e4ee86e' - pull/use image with SHA
         'breeze \
               --github-image-id 9a621eaa394c0a0a336f8e1b31b35eff4e4ee86e' - pull/use image with SHA
 

diff --git a/CI.rst b/CI.rst
@@ -78,7 +78,7 @@ this image can be built only once and used by all the jobs running tests. The im
 rather than build it from the scratch. Pulling such image takes ~ 1 minute, thanks to that we are saving
 a lot of precious time for jobs.
 
-We use `GitHub Container Registry <https://docs.github.com/en/packages/guides/about-github-container-registry>`_
+We use `GitHub Container Registry <https://docs.github.com/en/packages/guides/about-github-container-registry>`_.
 ``GITHUB_TOKEN`` is needed to push to the registry and we configured scopes of the tokens in our jobs
 to be able to write to the registry.
 
@@ -411,9 +411,9 @@ The following components are part of the CI infrastructure
 * **GitHub Actions** -  (GA) UI + execution engine for our jobs
 * **GA CRON trigger** - GitHub Actions CRON triggering our jobs
 * **GA Workers** - virtual machines running our jobs at GitHub Actions (max 20 in parallel)
-* **GitHub Image Registry**- image registry used as build cache for CI  jobs.
+* **GitHub Image Registry** - image registry used as build cache for CI jobs.
   It is at https://ghcr.io/apache/airflow
-* **DockerHub Image Registry**- image registry used to pull base Python images and (manually) publish
+* **DockerHub Image Registry** - image registry used to pull base Python images and (manually) publish
   the released Production Airflow images. It is at https://dockerhub.com/apache/airflow
 * **Official Images** (future) - these are official images that are prominently visible in DockerHub.
   We aim our images to become official images so that you will be able to pull them

diff --git a/breeze b/breeze
@@ -1688,7 +1688,7 @@ ${CMDNAME} shell [FLAGS] [-- <EXTRA_ARGS>]
       you want to pull the image with specific COMMIT_SHA tag.
 
       '${CMDNAME} shell \\
-            -- github-image-id 9a621eaa394c0a0a336f8e1b31b35eff4e4ee86e' - pull/use image with SHA
+            --github-image-id 9a621eaa394c0a0a336f8e1b31b35eff4e4ee86e' - pull/use image with SHA
       '${CMDNAME} \\
             --github-image-id 9a621eaa394c0a0a336f8e1b31b35eff4e4ee86e' - pull/use image with SHA
 

diff --git a/kubernetes_tests/test_kubernetes_pod_operator.py b/kubernetes_tests/test_kubernetes_pod_operator.py
@@ -867,7 +867,7 @@ def test_pod_template_file(self, mock_client, monitor_mock, start_mock):
                         'command': ['stress'],
                         'env': [],
                         'envFrom': [],
-                        'image': 'apache/airflow-ci:stress-2021.04.28-1.0.4',
+                        'image': 'ghcr.io/apache/airflow-stress:1.0.4-2021.07.04',
                         'name': 'base',
                         'ports': [],
                         'resources': {'limits': {'memory': '200Mi'}, 'requests': {'memory': '100Mi'}},

diff --git a/scripts/ci/docker-compose/integration-kerberos.yml b/scripts/ci/docker-compose/integration-kerberos.yml
@@ -18,7 +18,7 @@
 version: "2.2"
 services:
   kdc-server-example-com:
-    image: apache/airflow-ci:krb5-kdc-server-2021.04.28
+    image: ghcr.io/apache/airflow-krb5-kdc-server:2021.07.04
     hostname: krb5-kdc-server-example-com
     domainname: example.com
     networks:

diff --git a/scripts/ci/docker-compose/integration-openldap.yml b/scripts/ci/docker-compose/integration-openldap.yml
@@ -18,7 +18,7 @@
 version: "2.2"
 services:
   openldap:
-    image: apache/airflow-ci:openldap-2020.07.10-2.4.50
+    image: ghcr.io/apache/airflow-openldap:2.4.50-2021.07.04
     command: "--copy-service"
     environment:
       - LDAP_DOMAIN=example.com

diff --git a/scripts/ci/docker-compose/integration-trino.yml b/scripts/ci/docker-compose/integration-trino.yml
@@ -18,7 +18,7 @@
 version: "2.2"
 services:
   trino:
-    image: apache/airflow-ci:trino-2021.04.28
+    image: ghcr.io/apache/airflow-trino:359-2021.07.04
     container_name: trino
     hostname: trino
     domainname: example.com

diff --git a/scripts/ci/dockerfiles/apache-rat/build_and_push.sh b/scripts/ci/dockerfiles/apache-rat/build_and_push.sh
@@ -16,30 +16,29 @@
 # specific language governing permissions and limitations
 # under the License.
 set -euo pipefail
-DOCKERHUB_USER=${DOCKERHUB_USER:="apache"}
-DOCKERHUB_REPO=${DOCKERHUB_REPO:="airflow-ci"}
-readonly DOCKERHUB_USER
-readonly DOCKERHUB_REPO
+GITHUB_REPOSITORY=${GITHUB_REPOSITORY:="apache/airflow"}
+readonly GITHUB_REPOSITORY
 
 APACHERAT_VERSION="0.13"
 readonly APACHERAT_VERSION
 
-AIRFLOW_APACHERAT_VERSION="2021.04.28"
+AIRFLOW_APACHERAT_VERSION="2021.07.04"
 readonly AIRFLOW_APACHERAT_VERSION
 
 COMMIT_SHA=$(git rev-parse HEAD)
 readonly COMMIT_SHA
 
 cd "$( dirname "${BASH_SOURCE[0]}" )" || exit 1
 
-TAG="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:apache-rat-${AIRFLOW_APACHERAT_VERSION}-${APACHERAT_VERSION}"
+TAG="ghcr.io/${GITHUB_REPOSITORY}-apache-rat:${APACHERAT_VERSION}-${AIRFLOW_APACHERAT_VERSION}"
 readonly TAG
 
 docker build . \
     --pull \
     --build-arg "APACHERAT_VERSION=${APACHERAT_VERSION}" \
     --build-arg "AIRFLOW_APACHERAT_VERSION=${AIRFLOW_APACHERAT_VERSION}" \
     --build-arg "COMMIT_SHA=${COMMIT_SHA}" \
+    --label "org.opencontainers.image.source=https://github.com/${GITHUB_REPOSITORY}" \
     --tag "${TAG}"
 
 docker push "${TAG}"
diff --git a/scripts/ci/dockerfiles/bats/build_and_push.sh b/scripts/ci/dockerfiles/bats/build_and_push.sh
@@ -16,10 +16,8 @@
 # specific language governing permissions and limitations
 # under the License.
 set -euo pipefail
-DOCKERHUB_USER=${DOCKERHUB_USER:="apache"}
-DOCKERHUB_REPO=${DOCKERHUB_REPO:="airflow-ci"}
-readonly DOCKERHUB_USER
-readonly DOCKERHUB_REPO
+GITHUB_REPOSITORY=${GITHUB_REPOSITORY:="apache/airflow"}
+readonly GITHUB_REPOSITORY
 
 BATS_VERSION="1.2.1"
 BATS_ASSERT_VERSION="2.0.0"
@@ -30,15 +28,15 @@ readonly BATS_ASSERT_VERSION
 readonly BATS_SUPPORT_VERSION
 readonly BATS_FILE_VERSION
 
-AIRFLOW_BATS_VERSION="2021.04.28"
+AIRFLOW_BATS_VERSION="2021.07.04"
 readonly AIRFLOW_BATS_VERSION
 
 COMMIT_SHA=$(git rev-parse HEAD)
 readonly COMMIT_SHA
 
 cd "$( dirname "${BASH_SOURCE[0]}" )" || exit 1
 
-TAG="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:bats-${AIRFLOW_BATS_VERSION}-${BATS_VERSION}"
+TAG="ghcr.io/${GITHUB_REPOSITORY}-bats:${BATS_VERSION}-${AIRFLOW_BATS_VERSION}"
 readonly TAG
 
 docker build . \
@@ -48,6 +46,7 @@ docker build . \
     --build-arg "BATS_FILE_VERSION=${BATS_FILE_VERSION}" \
     --build-arg "BATS_ASSERT_VERSION=${BATS_ASSERT_VERSION}" \
     --build-arg "COMMIT_SHA=${COMMIT_SHA}" \
+    --label "org.opencontainers.image.source=https://github.com/${GITHUB_REPOSITORY}" \
     --tag "${TAG}"
 
 docker push "${TAG}"
diff --git a/scripts/ci/dockerfiles/krb5-kdc-server/build_and_push.sh b/scripts/ci/dockerfiles/krb5-kdc-server/build_and_push.sh
@@ -16,26 +16,25 @@
 # specific language governing permissions and limitations
 # under the License.
 set -euo pipefail
-DOCKERHUB_USER=${DOCKERHUB_USER:="apache"}
-DOCKERHUB_REPO=${DOCKERHUB_REPO:="airflow-ci"}
-readonly DOCKERHUB_USER
-readonly DOCKERHUB_REPO
+GITHUB_REPOSITORY=${GITHUB_REPOSITORY:="apache/airflow"}
+readonly GITHUB_REPOSITORY
 
-AIRFLOW_KRB5KDCSERVER_VERSION="2021.04.28"
+AIRFLOW_KRB5KDCSERVER_VERSION="2021.07.04"
 readonly AIRFLOW_KRB5KDCSERVER_VERSION
 
 COMMIT_SHA=$(git rev-parse HEAD)
 readonly COMMIT_SHA
 
 cd "$( dirname "${BASH_SOURCE[0]}" )" || exit 1
 
-TAG="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:krb5-kdc-server-${AIRFLOW_KRB5KDCSERVER_VERSION}"
+TAG="ghcr.io/${GITHUB_REPOSITORY}-krb5-kdc-server:${AIRFLOW_KRB5KDCSERVER_VERSION}"
 readonly TAG
 
 docker build . \
     --pull \
     --build-arg "AIRFLOW_KRB5KDCSERVER_VERSION=${AIRFLOW_KRB5KDCSERVER_VERSION}" \
     --build-arg "COMMIT_SHA=${COMMIT_SHA}" \
+    --label "org.opencontainers.image.source=https://github.com/${GITHUB_REPOSITORY}" \
     --tag "${TAG}"
 
 docker push "${TAG}"
diff --git a/scripts/ci/dockerfiles/stress/build_and_push.sh b/scripts/ci/dockerfiles/stress/build_and_push.sh
@@ -16,29 +16,29 @@
 # specific language governing permissions and limitations
 # under the License.
 set -euo pipefail
-DOCKERHUB_USER=${DOCKERHUB_USER:="apache"}
-DOCKERHUB_REPO=${DOCKERHUB_REPO:="airflow-ci"}
-readonly DOCKERHUB_USER
-readonly DOCKERHUB_REPO
+GITHUB_REPOSITORY=${GITHUB_REPOSITORY:="apache/airflow"}
+readonly GITHUB_REPOSITORY
+
 STRESS_VERSION="1.0.4"
 readonly STRESS_VERSION
 
-AIRFLOW_STRESS_VERSION="2021.04.28"
+AIRFLOW_STRESS_VERSION="2021.07.04"
 readonly AIRFLOW_STRESS_VERSION
 
 COMMIT_SHA=$(git rev-parse HEAD)
 readonly COMMIT_SHA
 
 cd "$( dirname "${BASH_SOURCE[0]}" )" || exit 1
 
-TAG="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:stress-${AIRFLOW_STRESS_VERSION}-${STRESS_VERSION}"
+TAG="ghcr.io/${GITHUB_REPOSITORY}-stress:${STRESS_VERSION}-${AIRFLOW_STRESS_VERSION}"
 readonly TAG
 
 docker build . \
     --pull \
     --build-arg "STRESS_VERSION=${STRESS_VERSION}" \
     --build-arg "AIRFLOW_STRESS_VERSION=${AIRFLOW_STRESS_VERSION}" \
     --build-arg "COMMIT_SHA=${COMMIT_SHA}" \
+    --label "org.opencontainers.image.source=https://github.com/${GITHUB_REPOSITORY}" \
     --tag "${TAG}"
 
 docker push "${TAG}"
diff --git a/scripts/ci/dockerfiles/trino/Dockerfile b/scripts/ci/dockerfiles/trino/Dockerfile
@@ -14,7 +14,7 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-ARG TRINO_VERSION="354"
+ARG TRINO_VERSION="359"
 FROM trinodb/trino:${TRINO_VERSION}
 
 # Obtain root privileges

diff --git a/scripts/ci/dockerfiles/trino/build_and_push.sh b/scripts/ci/dockerfiles/trino/build_and_push.sh
@@ -16,30 +16,29 @@
 # specific language governing permissions and limitations
 # under the License.
 set -euo pipefail
-DOCKERHUB_USER=${DOCKERHUB_USER:="apache"}
-DOCKERHUB_REPO=${DOCKERHUB_REPO:="airflow-ci"}
-readonly DOCKERHUB_USER
-readonly DOCKERHUB_REPO
+GITHUB_REPOSITORY=${GITHUB_REPOSITORY:="apache/airflow"}
+readonly GITHUB_REPOSITORY
 
-TRINO_VERSION="354"
+TRINO_VERSION="359"
 readonly TRINO_VERSION
 
-AIRFLOW_TRINO_VERSION="2021.04.28"
+AIRFLOW_TRINO_VERSION="2021.07.04"
 readonly AIRFLOW_TRINO_VERSION
 
 COMMIT_SHA=$(git rev-parse HEAD)
 readonly COMMIT_SHA
 
 cd "$( dirname "${BASH_SOURCE[0]}" )" || exit 1
 
-TAG="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:trino-${AIRFLOW_TRINO_VERSION}"
+TAG="ghcr.io/${GITHUB_REPOSITORY}-trino:${TRINO_VERSION}-${AIRFLOW_TRINO_VERSION}"
 readonly TAG
 
 docker build . \
     --pull \
     --build-arg "TRINO_VERSION=${TRINO_VERSION}" \
     --build-arg "AIRFLOW_TRINO_VERSION=${AIRFLOW_TRINO_VERSION}" \
     --build-arg "COMMIT_SHA=${COMMIT_SHA}" \
+    --label "org.opencontainers.image.source=https://github.com/${GITHUB_REPOSITORY}" \
     --tag "${TAG}"
 
 docker push "${TAG}"
diff --git a/scripts/ci/static_checks/bats_tests.sh b/scripts/ci/static_checks/bats_tests.sh
@@ -50,19 +50,18 @@ function run_bats_tests() {
         fi
 
     done
+    local airflow_bats_image="ghcr.io/apache/airflow-bats:1.2.1-2021.07.04"
     # deduplicate
     FS=" " read -r -a bats_arguments <<< "$(tr ' ' '\n' <<< "${bats_arguments[@]}" | sort -u | tr '\n' ' ' )"
     if [[ ${#@} == "0" ]]; then
         # Run al tests
-        docker run --workdir /airflow -v "$(pwd):/airflow" --rm \
-            apache/airflow-ci:bats-2021.04.28-1.2.1 --tap /airflow/tests/bats/
+        docker run --workdir /airflow -v "$(pwd):/airflow" --rm "${airflow_bats_image}" --tap /airflow/tests/bats/
     elif [[ ${#bats_arguments} == "0" ]]; then
         # Skip running anything if all filtered out
         true
     else
         # Run selected tests
-        docker run --workdir /airflow -v "$(pwd):/airflow" --rm \
-            apache/airflow-ci:bats-2021.04.28-1.2.1 --tap "${bats_arguments[@]}"
+        docker run --workdir /airflow -v "$(pwd):/airflow" --rm "${airflow_bats_image}" --tap "${bats_arguments[@]}"
     fi
 }
 

diff --git a/scripts/ci/static_checks/check_license.sh b/scripts/ci/static_checks/check_license.sh
@@ -34,7 +34,7 @@ function run_check_license() {
     if ! docker_v run -v "${AIRFLOW_SOURCES}:/opt/airflow" -t \
             --user "$(id -ur):$(id -gr)" \
             --rm --env-file "${AIRFLOW_SOURCES}/scripts/ci/docker-compose/_docker.env" \
-            apache/airflow-ci:apache-rat-2021.04.28-0.13 \
+            ghcr.io/apache/airflow-apache-rat:0.13-2021.07.04 \
             --exclude-file /opt/airflow/.rat-excludes \
             --d /opt/airflow | tee "${AIRFLOW_SOURCES}/logs/rat-results.txt" ; then
         echo

diff --git a/tests/kubernetes/pod.yaml b/tests/kubernetes/pod.yaml
@@ -23,7 +23,7 @@ metadata:
 spec:
   containers:
     - name: memory-demo-ctr
-      image: apache/airflow-ci:stress-2021.04.28-1.0.4
+      image: ghcr.io/apache/airflow-stress:1.0.4-2021.07.04
       resources:
         limits:
           memory: "200Mi"

diff --git a/tests/kubernetes/test_pod_generator.py b/tests/kubernetes/test_pod_generator.py
@@ -50,7 +50,7 @@ def setUp(self):
                     {
                         'args': ['--vm', '1', '--vm-bytes', '150M', '--vm-hang', '1'],
                         'command': ['stress'],
-                        'image': 'apache/airflow-ci:stress-2021.04.28-1.0.4',
+                        'image': 'ghcr.io/apache/airflow-stress:1.0.4-2021.07.04',
                         'name': 'memory-demo-ctr',
                         'resources': {'limits': {'memory': '200Mi'}, 'requests': {'memory': '100Mi'}},
                     }
@@ -688,7 +688,7 @@ def test_deserialize_model_string(self):
 spec:
   containers:
     - name: memory-demo-ctr
-      image: apache/airflow-ci:stress-2021.04.28-1.0.4
+      image: ghcr.io/apache/airflow-stress:1.0.4-2021.07.04
       resources:
         limits:
           memory: "200Mi"