Add --executor option to breeze kind-cluster deploy command (#15661)

This change will enable us easily deploy airflow to kubernetes cluster
and test it using different executors.
Example usage:
   ./breeze kind-cluster --executor CeleryExecutor deploy

GitOrigin-RevId: e47f7e42b632ad78a204531e385ec09bcce10816

BREEZE.rst

@@ -2531,6 +2531,14 @@ This is the current syntax for  `./breeze <./breeze>`_:
 
           Default: v3.2.4
 
+  --executor EXECUTOR
+          Executor to use in a kubernetes cluster.
+          One of:
+
+                 KubernetesExecutor CeleryExecutor LocalExecutor CeleryKubernetesExecutor
+
+          Default: KubernetesExecutor
+
   ****************************************************************************************************
    Manage mounting local files
 

breeze

@@ -639,6 +639,7 @@ export AIRFLOW_IMAGE="${airflow_image}"
 export SQLITE_URL="${SQLITE_URL}"
 export USE_AIRFLOW_VERSION="${USE_AIRFLOW_VERSION}"
 export USE_PACKAGES_FROM_DIST="${USE_PACKAGES_FROM_DIST}"
+export EXECUTOR = "${EXECUTOR}"
 docker-compose --log-level INFO ${command}
 EOF
     chmod u+x "${file}"
@@ -845,6 +846,12 @@ function breeze::parse_arguments() {
             echo
             shift 2
             ;;
+        --executor)
+            export EXECUTOR="${2}"
+            echo "Using ${EXECUTOR} in cluster"
+            echo
+            shift 2
+            ;;
         --helm-version)
             export HELM_VERSION="${2}"
             echo "Helm version: ${HELM_VERSION}"
@@ -1568,6 +1575,10 @@ function breeze::prepare_formatted_versions() {
         fold -w "${indented_screen_width}" -s | sed "s/^/${list_prefix}/")
     readonly FORMATTED_HELM_VERSIONS
 
+    FORMATTED_EXECUTORS=$(echo "${_breeze_allowed_executors=""}" | tr '\n' ' ' |
+        fold -w "${indented_screen_width}" -s | sed "s/^/${list_prefix}/")
+    readonly FORMATTED_EXECUTORS
+
     FORMATTED_KIND_OPERATIONS=$(echo "${_breeze_allowed_kind_operations=""}" | tr '\n' ' ' |
         fold -w "${indented_screen_width}" -s | sed "s/^/${list_prefix}/")
     readonly FORMATTED_KIND_OPERATIONS
@@ -2340,6 +2351,7 @@ ${FORMATTED_INTEGRATIONS}
 #       FORMATTED_KUBERNETES_VERSIONS
 #       FORMATTED_KIND_VERSIONS
 #       FORMATTED_HELM_VERSIONS
+#       FORMATTED_EXECUTORS
 #
 #
 # Outputs:
@@ -2381,6 +2393,14 @@ ${FORMATTED_HELM_VERSIONS}
 
         Default: ${_breeze_default_helm_version:=}
 
+--executor EXECUTOR
+        Executor to use in a kubernetes cluster.
+        One of:
+
+${FORMATTED_EXECUTORS}
+
+        Default: ${_breeze_default_executor:=}
+
 "
 }
 
@@ -3021,6 +3041,7 @@ function breeze::print_star_line() {
 #   KUBERNETES_VERSION
 #   KIND_VERSION
 #   HELM_VERSION
+#   EXECUTOR
 #   POSTGRES_VERSION
 #   MYSQL_VERSION
 #   DOCKERHUB_USER
@@ -3044,6 +3065,9 @@ function breeze::read_saved_environment_variables() {
     HELM_VERSION="${HELM_VERSION:=$(parameters::read_from_file HELM_VERSION)}"
     HELM_VERSION=${HELM_VERSION:=${_breeze_default_helm_version}}
 
+    EXECUTOR="${EXECUTOR:=$(parameters::read_from_file EXECUTOR)}"
+    EXECUTOR=${EXECUTOR:-${_breeze_default_executor}}
+
     POSTGRES_VERSION="${POSTGRES_VERSION:=$(parameters::read_from_file POSTGRES_VERSION)}"
     POSTGRES_VERSION=${POSTGRES_VERSION:=${_breeze_default_postgres_version}}
 
@@ -3080,6 +3104,7 @@ function breeze::read_saved_environment_variables() {
 #     KUBERNETES_VERSION
 #     KIND_VERSION
 #     HELM_VERSION
+#     EXECUTOR
 #     POSTGRES_VERSION
 #     MYSQL_VERSION
 #     DOCKERHUB_USER
@@ -3125,6 +3150,7 @@ function breeze::check_and_save_all_params() {
     parameters::check_and_save_allowed_param "KUBERNETES_VERSION" "Kubernetes version" "--kubernetes-version"
     parameters::check_and_save_allowed_param "KIND_VERSION" "KinD version" "--kind-version"
     parameters::check_and_save_allowed_param "HELM_VERSION" "Helm version" "--helm-version"
+    parameters::check_and_save_allowed_param "EXECUTOR" "Executors" "--executor"
     parameters::check_and_save_allowed_param "POSTGRES_VERSION" "Postgres version" "--postgres-version"
     parameters::check_and_save_allowed_param "MYSQL_VERSION" "Mysql version" "--mysql-version"
     parameters::check_and_save_allowed_param "GITHUB_REGISTRY" "GitHub Registry" "--github-registry"

breeze-complete

@@ -36,6 +36,7 @@ _breeze_allowed_kind_versions="v0.10.0"
 _breeze_allowed_mysql_versions="5.7 8"
 _breeze_allowed_postgres_versions="9.6 10 11 12 13"
 _breeze_allowed_kind_operations="start stop restart status deploy test shell k9s"
+_breeze_allowed_executors="KubernetesExecutor CeleryExecutor LocalExecutor CeleryKubernetesExecutor"
 _breeze_allowed_test_types="All Core Providers API CLI Integration Other WWW Postgres MySQL Helm Quarantined"
 _breeze_allowed_package_formats="both sdist wheel"
 _breeze_allowed_installation_methods=". apache-airflow"
@@ -50,6 +51,7 @@ _breeze_allowed_installation_methods=". apache-airflow"
     _breeze_default_kubernetes_version=$(echo "${_breeze_allowed_kubernetes_versions}" | awk '{print $1}')
     _breeze_default_helm_version=$(echo "${_breeze_allowed_helm_versions}" | awk '{print $1}')
     _breeze_default_kind_version=$(echo "${_breeze_allowed_kind_versions}" | awk '{print $1}')
+    _breeze_default_executor=$(echo "${_breeze_allowed_executors}" | awk '{print $1}')
     _breeze_default_postgres_version=$(echo "${_breeze_allowed_postgres_versions}" | awk '{print $1}')
     _breeze_default_mysql_version=$(echo "${_breeze_allowed_mysql_versions}" | awk '{print $1}')
     _breeze_default_test_type=$(echo "${_breeze_allowed_test_types}" | awk '{print $1}')
@@ -185,6 +187,7 @@ use-packages-from-dist no-rbac-ui package-format: upgrade-to-newer-dependencies
 use-airflow-version:
 cleanup-docker-context-files
 test-type: preserve-volumes dry-run-docker
+executor:
 "
 
 _breeze_commands="
@@ -305,6 +308,9 @@ function breeze_complete::get_known_values_breeze() {
     --test-type)
         _breeze_known_values="${_breeze_allowed_test_types}"
         ;;
+    --executor)
+      _breeze_known_values="${_breeze_allowed_executors}"
+      ;;
     --package-format)
         _breeze_known_values="${_breeze_allowed_package_formats}"
         ;;

scripts/ci/libraries/_initialization.sh

@@ -498,6 +498,9 @@ function initialization::initialize_kubernetes_variables() {
     # Currently supported versions of Helm
     CURRENT_HELM_VERSIONS+=("v3.2.4")
     export CURRENT_HELM_VERSIONS
+    # Current executor in chart
+    CURRENT_EXECUTOR+=("KubernetesExecutor")
+    export CURRENT_EXECUTOR
     # Default Kubernetes version
     export DEFAULT_KUBERNETES_VERSION="${CURRENT_KUBERNETES_VERSIONS[0]}"
     # Default Kubernetes mode
@@ -506,6 +509,8 @@ function initialization::initialize_kubernetes_variables() {
     export DEFAULT_KIND_VERSION="${CURRENT_KIND_VERSIONS[0]}"
     # Default Helm version
     export DEFAULT_HELM_VERSION="${CURRENT_HELM_VERSIONS[0]}"
+    # Default airflow executor used in cluster
+    export DEFAULT_EXECUTOR="${CURRENT_EXECUTOR[0]}"
     # Namespace where airflow is installed via helm
     export HELM_AIRFLOW_NAMESPACE="airflow"
     # Kubernetes version
@@ -516,6 +521,8 @@ function initialization::initialize_kubernetes_variables() {
     export KIND_VERSION=${KIND_VERSION:=${DEFAULT_KIND_VERSION}}
     # Helm version
     export HELM_VERSION=${HELM_VERSION:=${DEFAULT_HELM_VERSION}}
+    # Airflow Executor
+    export EXECUTOR=${EXECUTOR:=${DEFAULT_EXECUTOR}}
     # Kubectl version
     export KUBECTL_VERSION=${KUBERNETES_VERSION:=${DEFAULT_KUBERNETES_VERSION}}
     # Local Kind path
@@ -790,6 +797,7 @@ function initialization::make_constants_read_only() {
     readonly KUBERNETES_VERSION
     readonly KIND_VERSION
     readonly HELM_VERSION
+    readonly EXECUTOR
     readonly KUBECTL_VERSION
 
     readonly POSTGRES_VERSION
@@ -899,6 +907,7 @@ function initialization::make_constants_read_only() {
     readonly CURRENT_MYSQL_VERSIONS
     readonly CURRENT_KIND_VERSIONS
     readonly CURRENT_HELM_VERSIONS
+    readonly CURRENT_EXECUTOR
     readonly ALL_PYTHON_MAJOR_MINOR_VERSIONS
 }
 

scripts/ci/libraries/_kind.sh

@@ -131,6 +131,10 @@ function kind::perform_kind_cluster_operation() {
     echo "Kubernetes mode: ${KUBERNETES_MODE}"
     echo
 
+    echo
+    echo "Executor: ${EXECUTOR}"
+    echo
+
     if [[ ${OPERATION} == "status" ]]; then
         if [[ ${ALL_CLUSTERS} == *"${KIND_CLUSTER_NAME}"* ]]; then
             echo
@@ -333,7 +337,7 @@ function kind::deploy_airflow_with_helm() {
         --set "defaultAirflowTag=${AIRFLOW_PROD_BASE_TAG}-kubernetes" -v 1 \
         --set "config.api.auth_backend=airflow.api.auth.backend.basic_auth" \
         --set "config.logging.logging_level=DEBUG" \
-        --set "executor=KubernetesExecutor"
+        --set "executor=${EXECUTOR}"
     echo
     popd > /dev/null 2>&1|| exit 1
 }