Allow to specify pip version via github URL in Docker/CI workflow

We've only supported to install `pip` from released packages with a version number, but since `pip` does not support RC candidates (as extensively discussed in pypa/pip#10882) we cannot use the release versions to do that. We still want to help `pip` maintainers and be able to test the versions they release as early as possible, so we add support to install `pip` in our toolchain from a GitHub URL. That will allow us to test new `pip` version as soon as designated branch of `pip` will contain something resembling a release candidate ready for testing.
potiuk · Jan 4, 2023 · 72added · 72added
1 parent 628a71a
commit 72added
Show file tree

Hide file tree

Showing 10 changed files with 71 additions and 75 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -325,20 +325,12 @@ COPY <<"EOF" /install_pip_version.sh
 
 : "${AIRFLOW_PIP_VERSION:?Should be set}"
 
-function install_pip_version() {
-    echo
-    echo "${COLOR_BLUE}Installing pip version ${AIRFLOW_PIP_VERSION}${COLOR_RESET}"
-    echo
-    pip install --disable-pip-version-check --no-cache-dir --upgrade "pip==${AIRFLOW_PIP_VERSION}" &&
-        mkdir -p ${HOME}/.local/bin
-}
-
 common::get_colors
 common::get_airflow_version_specification
 common::override_pip_version_if_needed
 common::show_pip_version_and_location
 
-install_pip_version
+common::install_pip_version
 EOF
 
 # The content below is automatically copied from scripts/docker/install_airflow_dependencies_from_branch_tip.sh
@@ -369,8 +361,7 @@ function install_airflow_dependencies_from_branch_tip() {
       ${ADDITIONAL_PIP_INSTALL_FLAGS} \
       "https://github.com/${AIRFLOW_REPO}/archive/${AIRFLOW_BRANCH}.tar.gz#egg=apache-airflow[${AIRFLOW_EXTRAS}]" \
       --constraint "${AIRFLOW_CONSTRAINTS_LOCATION}" || true
-    # make sure correct PIP version is used
-    pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null
+    common::install_pip_version
     pip freeze | grep apache-airflow-providers | xargs pip uninstall --yes 2>/dev/null || true
     set +x
     echo
@@ -445,6 +436,18 @@ function common::show_pip_version_and_location() {
    echo "pip on path: $(which pip)"
    echo "Using pip: $(pip --version)"
 }
+
+function common::install_pip_version() {
+    echo
+    echo "${COLOR_BLUE}Installing pip version ${AIRFLOW_PIP_VERSION}${COLOR_RESET}"
+    echo
+    if [[ ${AIRFLOW_PIP_VERSION} =~ .*https.* ]]; then
+        pip install --disable-pip-version-check --no-cache-dir "pip @ ${AIRFLOW_PIP_VERSION}"
+    else
+        pip install --disable-pip-version-check --no-cache-dir "pip==${AIRFLOW_PIP_VERSION}"
+    fi
+    mkdir -p "${HOME}/.local/bin"
+}
 EOF
 
 # The content below is automatically copied from scripts/docker/pip
@@ -531,8 +534,7 @@ function install_airflow_and_providers_from_docker_context_files(){
         ${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS}
     set +x
 
-    # make sure correct PIP version is left installed
-    pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null
+    common::install_pip_version
     pip check
 }
 
@@ -549,9 +551,8 @@ function install_all_other_packages_from_docker_context_files() {
         set -x
         pip install ${ADDITIONAL_PIP_INSTALL_FLAGS} \
             --root-user-action ignore --force-reinstall --no-deps --no-index ${reinstalling_other_packages}
-        # make sure correct PIP version is used
-        pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null
-        set -x
+        common::install_pip_version
+        set +x
     fi
 }
 
@@ -613,8 +614,7 @@ function install_airflow() {
             set +x
         fi
 
-        # make sure correct PIP version is used
-        pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null
+        common::install_pip_version
         echo
         echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}"
         echo
@@ -628,15 +628,13 @@ function install_airflow() {
             ${ADDITIONAL_PIP_INSTALL_FLAGS} \
             "${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" \
             --constraint "${AIRFLOW_CONSTRAINTS_LOCATION}"
-        # make sure correct PIP version is used
-        pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null
+        common::install_pip_version
         # then upgrade if needed without using constraints to account for new limits in setup.py
         pip install --root-user-action ignore --upgrade --upgrade-strategy only-if-needed \
             ${ADDITIONAL_PIP_INSTALL_FLAGS} \
             ${AIRFLOW_INSTALL_EDITABLE_FLAG} \
             "${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}"
-        # make sure correct PIP version is used
-        pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null
+        common::install_pip_version
         set +x
         echo
         echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}"
@@ -675,8 +673,7 @@ function install_additional_dependencies() {
         pip install --root-user-action ignore --upgrade --upgrade-strategy eager \
             ${ADDITIONAL_PIP_INSTALL_FLAGS} \
             ${ADDITIONAL_PYTHON_DEPS} ${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS}
-        # make sure correct PIP version is used
-        pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null
+        common::install_pip_version
         set +x
         echo
         echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}"
@@ -690,8 +687,7 @@ function install_additional_dependencies() {
         pip install --root-user-action ignore --upgrade --upgrade-strategy only-if-needed \
             ${ADDITIONAL_PIP_INSTALL_FLAGS} \
             ${ADDITIONAL_PYTHON_DEPS}
-        # make sure correct PIP version is used
-        pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null
+        common::install_pip_version
         set +x
         echo
         echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}"

diff --git a/Dockerfile.ci b/Dockerfile.ci
@@ -285,20 +285,12 @@ COPY <<"EOF" /install_pip_version.sh
 
 : "${AIRFLOW_PIP_VERSION:?Should be set}"
 
-function install_pip_version() {
-    echo
-    echo "${COLOR_BLUE}Installing pip version ${AIRFLOW_PIP_VERSION}${COLOR_RESET}"
-    echo
-    pip install --disable-pip-version-check --no-cache-dir --upgrade "pip==${AIRFLOW_PIP_VERSION}" &&
-        mkdir -p ${HOME}/.local/bin
-}
-
 common::get_colors
 common::get_airflow_version_specification
 common::override_pip_version_if_needed
 common::show_pip_version_and_location
 
-install_pip_version
+common::install_pip_version
 EOF
 
 # The content below is automatically copied from scripts/docker/install_airflow_dependencies_from_branch_tip.sh
@@ -329,8 +321,7 @@ function install_airflow_dependencies_from_branch_tip() {
       ${ADDITIONAL_PIP_INSTALL_FLAGS} \
       "https://github.com/${AIRFLOW_REPO}/archive/${AIRFLOW_BRANCH}.tar.gz#egg=apache-airflow[${AIRFLOW_EXTRAS}]" \
       --constraint "${AIRFLOW_CONSTRAINTS_LOCATION}" || true
-    # make sure correct PIP version is used
-    pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null
+    common::install_pip_version
     pip freeze | grep apache-airflow-providers | xargs pip uninstall --yes 2>/dev/null || true
     set +x
     echo
@@ -405,6 +396,18 @@ function common::show_pip_version_and_location() {
    echo "pip on path: $(which pip)"
    echo "Using pip: $(pip --version)"
 }
+
+function common::install_pip_version() {
+    echo
+    echo "${COLOR_BLUE}Installing pip version ${AIRFLOW_PIP_VERSION}${COLOR_RESET}"
+    echo
+    if [[ ${AIRFLOW_PIP_VERSION} =~ .*https.* ]]; then
+        pip install --disable-pip-version-check --no-cache-dir "pip @ ${AIRFLOW_PIP_VERSION}"
+    else
+        pip install --disable-pip-version-check --no-cache-dir "pip==${AIRFLOW_PIP_VERSION}"
+    fi
+    mkdir -p "${HOME}/.local/bin"
+}
 EOF
 
 # The content below is automatically copied from scripts/docker/install_pipx_tools.sh
@@ -480,8 +483,7 @@ function install_airflow() {
             set +x
         fi
 
-        # make sure correct PIP version is used
-        pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null
+        common::install_pip_version
         echo
         echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}"
         echo
@@ -495,15 +497,13 @@ function install_airflow() {
             ${ADDITIONAL_PIP_INSTALL_FLAGS} \
             "${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" \
             --constraint "${AIRFLOW_CONSTRAINTS_LOCATION}"
-        # make sure correct PIP version is used
-        pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null
+        common::install_pip_version
         # then upgrade if needed without using constraints to account for new limits in setup.py
         pip install --root-user-action ignore --upgrade --upgrade-strategy only-if-needed \
             ${ADDITIONAL_PIP_INSTALL_FLAGS} \
             ${AIRFLOW_INSTALL_EDITABLE_FLAG} \
             "${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}"
-        # make sure correct PIP version is used
-        pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null
+        common::install_pip_version
         set +x
         echo
         echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}"
@@ -542,8 +542,7 @@ function install_additional_dependencies() {
         pip install --root-user-action ignore --upgrade --upgrade-strategy eager \
             ${ADDITIONAL_PIP_INSTALL_FLAGS} \
             ${ADDITIONAL_PYTHON_DEPS} ${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS}
-        # make sure correct PIP version is used
-        pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null
+        common::install_pip_version
         set +x
         echo
         echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}"
@@ -557,8 +556,7 @@ function install_additional_dependencies() {
         pip install --root-user-action ignore --upgrade --upgrade-strategy only-if-needed \
             ${ADDITIONAL_PIP_INSTALL_FLAGS} \
             ${ADDITIONAL_PYTHON_DEPS}
-        # make sure correct PIP version is used
-        pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null
+        common::install_pip_version
         set +x
         echo
         echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}"

diff --git a/scripts/docker/common.sh b/scripts/docker/common.sh
@@ -70,3 +70,15 @@ function common::show_pip_version_and_location() {
    echo "pip on path: $(which pip)"
    echo "Using pip: $(pip --version)"
 }
+
+function common::install_pip_version() {
+    echo
+    echo "${COLOR_BLUE}Installing pip version ${AIRFLOW_PIP_VERSION}${COLOR_RESET}"
+    echo
+    if [[ ${AIRFLOW_PIP_VERSION} =~ .*https.* ]]; then
+        pip install --disable-pip-version-check --no-cache-dir "pip @ ${AIRFLOW_PIP_VERSION}"
+    else
+        pip install --disable-pip-version-check --no-cache-dir "pip==${AIRFLOW_PIP_VERSION}"
+    fi
+    mkdir -p "${HOME}/.local/bin"
+}
diff --git a/scripts/docker/install_additional_dependencies.sh b/scripts/docker/install_additional_dependencies.sh
@@ -35,8 +35,7 @@ function install_additional_dependencies() {
         pip install --root-user-action ignore --upgrade --upgrade-strategy eager \
             ${ADDITIONAL_PIP_INSTALL_FLAGS} \
             ${ADDITIONAL_PYTHON_DEPS} ${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS}
-        # make sure correct PIP version is used
-        pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null
+        common::install_pip_version
         set +x
         echo
         echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}"
@@ -50,8 +49,7 @@ function install_additional_dependencies() {
         pip install --root-user-action ignore --upgrade --upgrade-strategy only-if-needed \
             ${ADDITIONAL_PIP_INSTALL_FLAGS} \
             ${ADDITIONAL_PYTHON_DEPS}
-        # make sure correct PIP version is used
-        pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null
+        common::install_pip_version
         set +x
         echo
         echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}"

diff --git a/scripts/docker/install_airflow.sh b/scripts/docker/install_airflow.sh
@@ -69,8 +69,7 @@ function install_airflow() {
             set +x
         fi
 
-        # make sure correct PIP version is used
-        pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null
+        common::install_pip_version
         echo
         echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}"
         echo
@@ -84,15 +83,13 @@ function install_airflow() {
             ${ADDITIONAL_PIP_INSTALL_FLAGS} \
             "${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" \
             --constraint "${AIRFLOW_CONSTRAINTS_LOCATION}"
-        # make sure correct PIP version is used
-        pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null
+        common::install_pip_version
         # then upgrade if needed without using constraints to account for new limits in setup.py
         pip install --root-user-action ignore --upgrade --upgrade-strategy only-if-needed \
             ${ADDITIONAL_PIP_INSTALL_FLAGS} \
             ${AIRFLOW_INSTALL_EDITABLE_FLAG} \
             "${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}"
-        # make sure correct PIP version is used
-        pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null
+        common::install_pip_version
         set +x
         echo
         echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}"

diff --git a/scripts/docker/install_airflow_dependencies_from_branch_tip.sh b/scripts/docker/install_airflow_dependencies_from_branch_tip.sh
@@ -52,8 +52,7 @@ function install_airflow_dependencies_from_branch_tip() {
       ${ADDITIONAL_PIP_INSTALL_FLAGS} \
       "https://github.com/${AIRFLOW_REPO}/archive/${AIRFLOW_BRANCH}.tar.gz#egg=apache-airflow[${AIRFLOW_EXTRAS}]" \
       --constraint "${AIRFLOW_CONSTRAINTS_LOCATION}" || true
-    # make sure correct PIP version is used
-    pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null
+    common::install_pip_version
     pip freeze | grep apache-airflow-providers | xargs pip uninstall --yes 2>/dev/null || true
     set +x
     echo

diff --git a/scripts/docker/install_from_docker_context_files.sh b/scripts/docker/install_from_docker_context_files.sh
@@ -85,8 +85,7 @@ function install_airflow_and_providers_from_docker_context_files(){
         ${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS}
     set +x
 
-    # make sure correct PIP version is left installed
-    pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null
+    common::install_pip_version
     pip check
 }
 
@@ -107,9 +106,8 @@ function install_all_other_packages_from_docker_context_files() {
         set -x
         pip install ${ADDITIONAL_PIP_INSTALL_FLAGS} \
             --root-user-action ignore --force-reinstall --no-deps --no-index ${reinstalling_other_packages}
-        # make sure correct PIP version is used
-        pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null
-        set -x
+        common::install_pip_version
+        set +x
     fi
 }
 

diff --git a/scripts/docker/install_pip_version.sh b/scripts/docker/install_pip_version.sh
@@ -20,17 +20,9 @@
 
 : "${AIRFLOW_PIP_VERSION:?Should be set}"
 
-function install_pip_version() {
-    echo
-    echo "${COLOR_BLUE}Installing pip version ${AIRFLOW_PIP_VERSION}${COLOR_RESET}"
-    echo
-    pip install --disable-pip-version-check --no-cache-dir --upgrade "pip==${AIRFLOW_PIP_VERSION}" &&
-        mkdir -p ${HOME}/.local/bin
-}
-
 common::get_colors
 common::get_airflow_version_specification
 common::override_pip_version_if_needed
 common::show_pip_version_and_location
 
-install_pip_version
+common::install_pip_version
diff --git a/scripts/in_container/_in_container_utils.sh b/scripts/in_container/_in_container_utils.sh
@@ -353,7 +353,12 @@ function setup_provider_packages() {
 
 
 function install_supported_pip_version() {
-    pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}"
+    if [[ ${AIRFLOW_PIP_VERSION} =~ .*https.* ]]; then
+        pip install --disable-pip-version-check "pip @ ${AIRFLOW_PIP_VERSION}"
+    else
+        pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}"
+    fi
+
 }
 
 function filename_to_python_module() {

diff --git a/scripts/in_container/run_prepare_airflow_packages.sh b/scripts/in_container/run_prepare_airflow_packages.sh
@@ -34,7 +34,8 @@ function prepare_airflow_packages() {
     rm -rf -- *egg-info*
     rm -rf -- build
 
-    pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" "wheel==${WHEEL_VERSION}"
+    install_supported_pip_version
+    pip install "wheel==${WHEEL_VERSION}"
 
     local packages=()