Remove legacy GitHub Packages (#16776)

This PR removes the legacy GitHub Packages support: * removes checking for images in Packages/Registry * removes output informing about the registry * hard-codes registry to ghcr.io * Updaes documentation describing the registries
apache · Jul 4, 2021 · b251d22 · b251d22
1 parent 5399f91
commit b251d22
Show file tree

Hide file tree

Showing 9 changed files with 38 additions and 246 deletions.
diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml
@@ -47,7 +47,6 @@ env:
   GITHUB_REGISTRY_PULL_IMAGE_TAG: "latest"
   GITHUB_REGISTRY_WAIT_FOR_IMAGE: "false"
   INSTALL_PROVIDERS_FROM_SOURCES: "true"
-  GITHUB_REGISTRY: "ghcr.io"
   TARGET_COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
 
 concurrency:

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -37,7 +37,6 @@ env:
   VERBOSE: "true"
   DOCKER_CACHE: "pulled"
   USE_GITHUB_REGISTRY: "true"
-  GITHUB_REGISTRY: "ghcr.io"
   GITHUB_REPOSITORY: ${{ github.repository }}
   GITHUB_USERNAME: ${{ github.actor }}
   # You can override CONSTRAINTS_GITHUB_REPOSITORY by setting secret in your repo but by default the
@@ -268,8 +267,6 @@ jobs:
       BACKEND: sqlite
       UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgradeToNewerDependencies }}
       WAIT_FOR_IMAGE: ${{ needs.build-info.outputs.waitForImage }}
-    outputs:
-      githubRegistry: ${{ steps.wait-for-images.outputs.githubRegistry }}
     steps:
       - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
         uses: actions/checkout@v2
@@ -297,9 +294,6 @@ jobs:
         # We are utilising single job to wait for all images because this job merely waits
         # for the images to be available.
         # The test jobs wait for it to complete if WAIT_FOR_IMAGE is 'true'!
-        # The job will set the output "githubRegistry" - result of auto-detect which registry has
-        # been used by checking where the image can be downloaded from.
-        #
         run: ./scripts/ci/images/ci_wait_for_and_verify_all_ci_images.sh
 
 
@@ -313,7 +307,6 @@ jobs:
       SKIP: "identity"
       MOUNT_SELECTED_LOCAL_SOURCES: "true"
       PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}}
-      GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }}
     if: needs.build-info.outputs.basic-checks-only == 'false'
     steps:
       - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
@@ -419,7 +412,6 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
     if: needs.build-info.outputs.docs-build == 'true'
     env:
       RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }}
-      GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }}
     steps:
       - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
         uses: actions/checkout@v2
@@ -467,7 +459,6 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
       AIRFLOW_EXTRAS: "all"
       PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}}
       VERSION_SUFFIX_FOR_PYPI: ".dev0"
-      GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }}
       NON_INTERACTIVE: "true"
       GENERATE_PROVIDERS_ISSUE: "true"
     if: needs.build-info.outputs.image-build == 'true' && needs.build-info.outputs.default-branch == 'main'
@@ -516,7 +507,6 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
       AIRFLOW_EXTRAS: "all"
       PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}}
       VERSION_SUFFIX_FOR_PYPI: ".dev0"
-      GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }}
       NON_INTERACTIVE: "true"
       GENERATE_PROVIDERS_ISSUE: "true"
     if: needs.build-info.outputs.image-build == 'true' && needs.build-info.outputs.default-branch == 'main'
@@ -560,7 +550,6 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
       BACKEND: ""
       DB_RESET: "false"
       PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}}
-      GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }}
     if: >
       needs.build-info.outputs.needs-helm-tests == 'true' &&
       (github.repository == 'apache/airflow' || github.event_name != 'schedule')
@@ -620,7 +609,6 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
       PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
       POSTGRES_VERSION: ${{ matrix.postgres-version }}
       TEST_TYPES: "${{needs.build-info.outputs.testTypes}}"
-      GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }}
     if: needs.build-info.outputs.run-tests == 'true'
     steps:
       - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
@@ -677,7 +665,6 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
       PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
       MYSQL_VERSION: ${{ matrix.mysql-version }}
       TEST_TYPES: "${{needs.build-info.outputs.testTypes}}"
-      GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }}
     if: needs.build-info.outputs.run-tests == 'true'
     steps:
       - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
@@ -733,7 +720,6 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
       PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
       MSSQL_VERSION: ${{ matrix.mssql-version }}
       TEST_TYPES: "${{needs.build-info.outputs.testTypes}}"
-      GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }}
     if: needs.build-info.outputs.run-tests == 'true'
     steps:
       - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
@@ -787,7 +773,6 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
       BACKEND: sqlite
       PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
       TEST_TYPES: "${{needs.build-info.outputs.testTypes}}"
-      GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }}
     if: needs.build-info.outputs.run-tests == 'true'
     steps:
       - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
@@ -839,7 +824,6 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
       TEST_TYPES: "Quarantined"
       NUM_RUNS: 10
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }}
     if: needs.build-info.outputs.run-tests == 'true'
     steps:
       - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
@@ -939,8 +923,6 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
       BACKEND: sqlite
       PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.defaultPythonVersion }}
       UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgradeToNewerDependencies }}
-    outputs:
-      githubRegistry: ${{ steps.wait-for-images.outputs.githubRegistry }}
     steps:
       - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
         uses: actions/checkout@v2
@@ -963,8 +945,6 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
         # "build-images-workflow-run.yml' run as pull_request_target.
         # We are utilising single job to wait for all images because this job merely waits
         # For the images to be available. The test jobs wait for it to complete!
-        # The job will set the output "githubRegistry" - result of auto-detect which registry has
-        # been used by checking where the image can be downloaded from.
         #
         id: wait-for-images
         env:
@@ -989,7 +969,6 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
       EXECUTOR: ${{matrix.executor}}
       KIND_VERSION: "${{ needs.build-info.outputs.defaultKindVersion }}"
       HELM_VERSION: "${{ needs.build-info.outputs.defaultHelmVersion }}"
-      GITHUB_REGISTRY: ${{ needs.prod-images.outputs.githubRegistry }}
       CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING: >
         ${{needs.build-info.outputs.pythonVersionsListAsString}}
       CURRENT_KUBERNETES_VERSIONS_AS_STRING: >
@@ -1055,7 +1034,6 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
       EXECUTOR: "KubernetesExecutor"
       KIND_VERSION: "${{ needs.build-info.outputs.defaultKindVersion }}"
       HELM_VERSION: "${{ needs.build-info.outputs.defaultHelmVersion }}"
-      GITHUB_REGISTRY: ${{ needs.prod-images.outputs.githubRegistry }}
       CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING: >
         ${{needs.build-info.outputs.pythonVersionsListAsString}}
       CURRENT_KUBERNETES_VERSIONS_AS_STRING: >
@@ -1131,7 +1109,6 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
       RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }}
       PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
       GITHUB_REGISTRY_PUSH_IMAGE_TAG: "latest"
-      GITHUB_REGISTRY: ${{ needs.prod-images.outputs.githubRegistry }}
     steps:
       - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
         uses: actions/checkout@v2
@@ -1193,7 +1170,6 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
       RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }}
       PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
       GITHUB_REGISTRY_PUSH_IMAGE_TAG: "latest"
-      GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }}
     steps:
       - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
         uses: actions/checkout@v2
@@ -1229,7 +1205,6 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
     env:
       RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }}
       PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
-      GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }}
       CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING: ${{needs.build-info.outputs.pythonVersionsListAsString}}
     # Only run it for direct pushes
     if: >
@@ -1326,8 +1301,6 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
     name: React UI tests
     runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
     needs: [build-info, ci-images]
-    env:
-      GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }}
     if: needs.build-info.outputs.run-ui-tests == 'true'
     steps:
       - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"

diff --git a/BREEZE.rst b/BREEZE.rst
@@ -1446,16 +1446,6 @@ This is the current syntax for  `./breeze <./breeze>`_:
           DockerHub. You need to be logged in to the registry in order to be able to pull/push from
           and you need to be committer to push to Apache Airflow' GitHub registry.
 
-  --github-registry GITHUB_REGISTRY
-          GitHub registry used. GitHub has legacy Packages registry and Public Beta Container
-          registry.
-
-          Default: ghcr.io.
-
-          If you use this flag, automatically --use-github-registry flag is enabled.
-
-                 ghcr.io docker.pkg.github.com
-
   -g, --github-repository GITHUB_REPOSITORY
           GitHub repository used to pull, push images when cache is used.
           Default: apache/airflow.
@@ -1620,16 +1610,6 @@ This is the current syntax for  `./breeze <./breeze>`_:
           DockerHub. You need to be logged in to the registry in order to be able to pull/push from
           and you need to be committer to push to Apache Airflow' GitHub registry.
 
-  --github-registry GITHUB_REGISTRY
-          GitHub registry used. GitHub has legacy Packages registry and Public Beta Container
-          registry.
-
-          Default: ghcr.io.
-
-          If you use this flag, automatically --use-github-registry flag is enabled.
-
-                 ghcr.io docker.pkg.github.com
-
   -g, --github-repository GITHUB_REPOSITORY
           GitHub repository used to pull, push images when cache is used.
           Default: apache/airflow.
@@ -2693,16 +2673,6 @@ This is the current syntax for  `./breeze <./breeze>`_:
           DockerHub. You need to be logged in to the registry in order to be able to pull/push from
           and you need to be committer to push to Apache Airflow' GitHub registry.
 
-  --github-registry GITHUB_REGISTRY
-          GitHub registry used. GitHub has legacy Packages registry and Public Beta Container
-          registry.
-
-          Default: ghcr.io.
-
-          If you use this flag, automatically --use-github-registry flag is enabled.
-
-                 ghcr.io docker.pkg.github.com
-
   -g, --github-repository GITHUB_REPOSITORY
           GitHub repository used to pull, push images when cache is used.
           Default: apache/airflow.

diff --git a/CI.rst b/CI.rst
@@ -54,7 +54,7 @@ it can be ~6-7 minutes and in case base image of Python releases new patch-level
 Container Registry used as cache
 --------------------------------
 
-For the CI builds of our we are using Container Registry to store results of the "Build Image" workflow
+For the CI builds of our we are using GitHub Container Registry to store results of the "Build Image" workflow
 and pass it to the "CI Build" workflow.
 
 Currently in main version of Airflow we run tests in 4 different versions of Python (3.6, 3.7, 3.8, 3.9)
@@ -70,40 +70,23 @@ This is especially important in our case where Pull Requests to Airflow might co
 and it would be a huge security issue if anyone from outside could
 utilise the WRITE access to Apache Airflow repository via an external Pull Request.
 
-Thanks to the WRITE access and fact that the 'pull_request_target' by default uses the 'main' version of the
+Thanks to the WRITE access and fact that the ``pull_request_target`` by default uses the ``main`` version of the
 sources, we can safely run some logic there will checkout the incoming Pull Request, build the container
 image from the sources from the incoming PR and push such image to an GitHub Docker Registry - so that
 this image can be built only once and used by all the jobs running tests. The image is tagged with unique
 ``COMMIT_SHA`` of the incoming Pull Request and the tests run in the Pull Request can simply pull such image
 rather than build it from the scratch. Pulling such image takes ~ 1 minute, thanks to that we are saving
 a lot of precious time for jobs.
+4
+We use `GitHub Container Registry <https://docs.github.com/en/packages/guides/about-github-container-registry>`_
+GitHub Package Registry ``GITHUB_TOKEN`` is needed to push to the registry. You also have to manually manage
+permissions of the images, after creating image for the first time (pushing it using your personal token)
+you need to set their visibility to ``Public`` and enable
+`Inheriting access from repository <https://docs.github.com/en/packages/learn-github-packages/configuring-a-packages-access-control-and-visibility#inheriting-access-for-a-container-image-from-a-repository>`_
+Those images have specific naming schema. See `Images documentation <IMAGES.rst>`_ for details.
 
-We can use either of the two available GitHub Container registries as cache:
-
-* Legacy `GitHub Package Registry <https://github.com/features/packages>`_ which is not very
-  stable, uses old infrastructure of GitHub and it lacks certain features - notably it does not allow
-  us to delete the old image. The benefit of using GitHub Package Registry is that it works
-  out-of-the-box (write authentication is done using ``GITHUB_TOKEN`` and users do not have to do any
-  action to make it work in case they want to run build using their own forks. Also those images
-  do not provide public access, so you need to login to ``docker.pkg.github.com`` docker registry
-  using your username and personal token to be able to pull those images.
-
-* The new `GitHub Container Registry <https://docs.github.com/en/packages/guides/about-github-container-registry>`_
-  which is in Public Beta, has many more features (including permission management, public access and
-  image retention possibility). Similarly as in case of GitHub Package Registry ``GITHUB_TOKEN`` is needed
-  to push to the repositories. You also have to manually manage permissions of the images,
-  i.e. after creating images for the first time, you need to set their visibility to ``Public`` and
-  add ``Admin`` permissions to group of people managing the images (in our case ``airflow-committers`` group).
-  This makes it not very suitable to use GitHub container registry if you want to run builds of Airflow
-  in your own forks (note - it does not affect pull requests from forks to Airflow).
-
-Those two images have different naming schemas. See `Images documentation <IMAGES.rst>`_ for details.
-
-You can interact with the GitHub Registry images (pull/push) via `Breeze <BREEZE.rst>`_  - you can
-pass ``--github-registry`` flag with either ``docker.pkg.github.com`` for GitHub Package Registry or
-``ghcr.io`` for GitHub Container Registry and pull/push operations will be performed using the chosen
-registry, using appropriate naming convention. This allows building and pushing the images locally by
-committers who have access to push/pull those images.
+You can interact with the GitHub Registry images (pull/push) via `Breeze <BREEZE.rst>`_  - by passing
+``--use-github-registry`` flag.
 
 Locally replicating CI failures
 -------------------------------
@@ -787,7 +770,7 @@ cd27124534b46c9688a1d89e75fcd137ab5137e3, in python 3.8 environment you can run:
 
 .. code-block:: bash
 
-  ./breeze --github-image-id cd27124534b46c9688a1d89e75fcd137ab5137e3 --github-registry ghcr.io --python 3.8
+  ./breeze --github-image-id cd27124534b46c9688a1d89e75fcd137ab5137e3 --use=github-registry --python 3.8
 
 You will be dropped into a shell with the exact version that was used during the CI run and you will
 be able to run pytest tests manually, easily reproducing the environment that was used in CI. Note that in
@@ -848,8 +831,7 @@ In order to add a new version the following operations should be done (example u
 .. code-block:: bash
 
   ./breeze push-image --python 3.9
-  ./breeze push-image --python 3.9 --github-registry ghcr.io
-  ./breeze push-image --python 3.9 --github-registry docker.pkg.github.com
+  ./breeze push-image --python 3.9 --use-github-registry
 
 * Find the 3 new images (main, ci, build) created in
   `GitHub Container registry <https://github.com/orgs/apache/packages?tab=packages&ecosystem=container&q=airflow>`_