Merge pull request ovn-kubernetes#2302 from pliurh/ds_merge

SDN-4930,OCPBUGS-39009: Downstream Merge Sept 18th

.github/workflows/test.yml

@@ -451,7 +451,7 @@ jobs:
       KIND_IPV4_SUPPORT: "${{ matrix.ipfamily == 'IPv4' || matrix.ipfamily == 'dualstack' }}"
       KIND_IPV6_SUPPORT: "${{ matrix.ipfamily == 'IPv6' || matrix.ipfamily == 'dualstack' }}"
       ENABLE_MULTI_NET: "${{ matrix.target == 'multi-homing' || matrix.target == 'kv-live-migration' || matrix.target == 'network-segmentation' || matrix.target == 'tools' || matrix.target == 'multi-homing-helm' }}"
-      ENABLE_NETWORK_SEGMENTATION: "${{ matrix.target == 'network-segmentation' || matrix.target == 'tools'}}"
+      ENABLE_NETWORK_SEGMENTATION: "${{ matrix.target == 'network-segmentation' || matrix.target == 'tools' || matrix.target == 'kv-live-migration'}}"
       KIND_INSTALL_KUBEVIRT: "${{ matrix.target == 'kv-live-migration' }}"
       OVN_COMPACT_MODE: "${{ matrix.target == 'compact-mode' }}"
       OVN_DUMMY_GATEWAY_BRIDGE: "${{ matrix.target == 'compact-mode' }}"
@@ -617,6 +617,17 @@ jobs:
         echo "GOPATH=$GOPATH" >> $GITHUB_ENV
         echo "$GOPATH/bin" >> $GITHUB_PATH
 
+    - name: Free up disk space
+      run: |
+        sudo rm -rf /usr/local/lib/android/sdk
+        sudo apt-get update
+        sudo eatmydata apt-get purge --auto-remove -y \
+          azure-cli aspnetcore-* dotnet-* ghc-* firefox \
+          google-chrome-stable \
+          llvm-* microsoft-edge-stable mono-* \
+          msbuild mysql-server-core-* php-* php7* \
+          powershell temurin-* zulu-*
+
     - name: Disable ufw
       # For IPv6 and Dualstack, ufw (Uncomplicated Firewall) should be disabled.
       # Not needed for KIND deployments, so just disable all the time.

Dockerfile

@@ -57,6 +57,7 @@ COPY --from=builder /go/src/github.com/openshift/ovn-kubernetes/go-controller/_o
 COPY --from=builder /go/src/github.com/openshift/ovn-kubernetes/go-controller/_output/go/bin/ovndbchecker /usr/bin/
 COPY --from=builder /go/src/github.com/openshift/ovn-kubernetes/go-controller/_output/go/bin/ovnkube-trace /usr/bin/
 COPY --from=builder /go/src/github.com/openshift/ovn-kubernetes/go-controller/_output/go/bin/hybrid-overlay-node /usr/bin/
+COPY --from=builder /go/src/github.com/openshift/ovn-kubernetes/go-controller/_output/go/bin/ovnkube-observ /usr/bin/
 
 # Copy RHEL-8 and RHEL-9 shim binaries where the CNO's ovnkube-node container startup script can find them
 RUN mkdir -p /usr/libexec/cni/rhel9

contrib/kind-common

@@ -122,12 +122,13 @@ install_ingress() {
 
 METALLB_DIR="/tmp/metallb"
 install_metallb() {
+  local metallb_version=v0.14.8
   mkdir -p /tmp/metallb
   local builddir
   builddir=$(mktemp -d "${METALLB_DIR}/XXXXXX")
 
   pushd "${builddir}"
-  git clone https://github.com/metallb/metallb.git
+  git clone https://github.com/metallb/metallb.git -b $metallb_version
   cd metallb
   # Use global IP next hops in IPv6
   if  [ "$KIND_IPV6_SUPPORT" == true ]; then
@@ -348,6 +349,11 @@ install_kubevirt() {
             kubectl logs --all-containers=true -n kubevirt $p || true
         done
     fi
+
+    kubectl -n kubevirt patch kubevirt kubevirt --type=json --patch '[{"op":"add","path":"/spec/configuration/developerConfiguration","value":{"featureGates":[]}},{"op":"add","path":"/spec/configuration/developerConfiguration/featureGates/-","value":"NetworkBindingPlugins"}]'
+
+    local passt_binding_image="quay.io/kubevirt/network-passt-binding:${kubevirt_version}"
+    kubectl -n kubevirt patch kubevirt kubevirt --type=json --patch '[{"op":"add","path":"/spec/configuration/network","value":{}},{"op":"add","path":"/spec/configuration/network/binding","value":{"passt":{"computeResourceOverhead":{"requests":{"memory":"500Mi"}},"migration":{"method":"link-refresh"},"networkAttachmentDefinition":"default/primary-udn-kubevirt-binding","sidecarImage":"'"${passt_binding_image}"'"}}}]'
 
     if [ ! -d "./bin" ]
     then
@@ -373,22 +379,26 @@ install_kubevirt() {
     chmod +x ./bin/virtctl
 }
 
-install_kubevirt_ipam_controller() {
+install_cert_manager() {
   local cert_manager_version="v1.14.4"
   echo "Installing cert-manager ..."
   manifest="https://github.com/cert-manager/cert-manager/releases/download/${cert_manager_version}/cert-manager.yaml"
   run_kubectl apply -f "$manifest"
+}
 
+install_kubevirt_ipam_controller() {
   echo "Installing KubeVirt IPAM controller manager ..."
   manifest="https://raw.githubusercontent.com/kubevirt/ipam-extensions/main/dist/install.yaml"
   run_kubectl apply -f "$manifest"
   kubectl wait -n kubevirt-ipam-controller-system deployment kubevirt-ipam-controller-manager --for condition=Available --timeout 2m
 }
 
 install_multus() {
-  echo "Installing multus-cni daemonset ..."
-  multus_manifest="https://raw.githubusercontent.com/k8snetworkplumbingwg/multus-cni/master/deployments/multus-daemonset.yml"
-  run_kubectl apply -f "$multus_manifest"
+  local version="v4.1.0"
+  echo "Installing multus-cni $version daemonset ..."
+  wget -qO- "https://raw.githubusercontent.com/k8snetworkplumbingwg/multus-cni/${version}/deployments/multus-daemonset.yml" |\
+    sed -e "s|multus-cni:snapshot|multus-cni:${version}|g" |\
+    run_kubectl apply -f -
 }
 
 install_mpolicy_crd() {
@@ -576,3 +586,32 @@ kubectl_wait_dnsnameresolver_pods() {
   echo "Waiting for pods in dnsnameresolver-operator namespace to become ready (timeout ${timeout})..."
   kubectl wait -n dnsnameresolver-operator --for=condition=ready pods --all --timeout=${timeout}s
 }
+
+deploy_kubevirt_binding() {
+  cat <<EOF | run_kubectl apply -f -
+---
+apiVersion: "k8s.cni.cncf.io/v1"
+kind: NetworkAttachmentDefinition
+metadata:
+  name: primary-udn-kubevirt-binding
+  namespace: default
+spec:
+  config: '{
+  "cniVersion": "1.0.0",
+  "name": "primary-udn-kubevirt-binding",
+  "plugins": [
+    {
+      "type": "network-passt-binding"
+    }
+  ]
+}'
+EOF
+}
+
+deploy_passt_binary() {
+  echo "Installing passt-binding-cni-ds ..."
+  local manifest="https://raw.githubusercontent.com/kubevirt/ipam-extensions/main/passt/passt-binding-cni-ds.yaml"
+  run_kubectl apply -f "$manifest"
+
+  run_kubectl rollout status -n kube-system daemonset/passt-binding-cni --timeout 2m
+}

contrib/kind-helm.sh

@@ -21,6 +21,7 @@ set_default_params() {
   export OVN_HA=${OVN_HA:-false}
   export OVN_MULTICAST_ENABLE=${OVN_MULTICAST_ENABLE:-false}
   export OVN_HYBRID_OVERLAY_ENABLE=${OVN_HYBRID_OVERLAY_ENABLE:-false}
+  export OVN_OBSERV_ENABLE=${OVN_OBSERV_ENABLE:-false}
   export OVN_EMPTY_LB_EVENTS=${OVN_EMPTY_LB_EVENTS:-false}
   export KIND_REMOVE_TAINT=${KIND_REMOVE_TAINT:-true}
   export ENABLE_MULTI_NET=${ENABLE_MULTI_NET:-false}
@@ -106,6 +107,7 @@ usage() {
     echo "                                    DEFAULT: Remove taint components"
     echo "-me  | --multicast-enabled          Enable multicast. DEFAULT: Disabled"
     echo "-ho  | --hybrid-enabled             Enable hybrid overlay. DEFAULT: Disabled"
+    echo "-obs | --observability              Enable observability. DEFAULT: Disabled"
     echo "-el  | --ovn-empty-lb-events        Enable empty-lb-events generation for LB without backends. DEFAULT: Disabled"
     echo "-ii  | --install-ingress            Flag to install Ingress Components."
     echo "                                    DEFAULT: Don't install ingress components."
@@ -143,6 +145,8 @@ parse_args() {
                                                 ;;
             -ho | --hybrid-enabled )            OVN_HYBRID_OVERLAY_ENABLE=true
                                                 ;;
+            -obs | --observability )            OVN_OBSERV_ENABLE=true
+                                                ;;
             -el | --ovn-empty-lb-events )       OVN_EMPTY_LB_EVENTS=true
                                                 ;;
             -ii | --install-ingress )           KIND_INSTALL_INGRESS=true
@@ -202,6 +206,7 @@ print_params() {
      echo "OVN_HA = $OVN_HA"
      echo "OVN_MULTICAST_ENABLE = $OVN_MULTICAST_ENABLE"
      echo "OVN_HYBRID_OVERLAY_ENABLE = $OVN_HYBRID_OVERLAY_ENABLE"
+     echo "OVN_OBSERV_ENABLE = $OVN_OBSERV_ENABLE"
      echo "OVN_EMPTY_LB_EVENTS = $OVN_EMPTY_LB_EVENTS"
      echo "KIND_CLUSTER_NAME = $KIND_CLUSTER_NAME"
      echo "KIND_REMOVE_TAINT = $KIND_REMOVE_TAINT"
@@ -398,7 +403,8 @@ create_ovn_kubernetes() {
           --set global.enableMulticast=$(if [ "${OVN_MULTICAST_ENABLE}" == "true" ]; then echo "true"; else echo "false"; fi) \
           --set global.enableMultiNetwork=$(if [ "${ENABLE_MULTI_NET}" == "true" ]; then echo "true"; else echo "false"; fi) \
           --set global.enableHybridOverlay=$(if [ "${OVN_HYBRID_OVERLAY_ENABLE}" == "true" ]; then echo "true"; else echo "false"; fi) \
-          --set global.emptyLbEvents=$(if [ "${OVN_EMPTY_LB_EVENTS}" == "true" ]; then echo "true"; else echo "false"; fi) \
+          --set global.enableObservability=$(if [ "${OVN_OBSERV_ENABLE}" == "true" ]; then echo "true"; else echo "false"; fi) \
+        --set global.emptyLbEvents=$(if [ "${OVN_EMPTY_LB_EVENTS}" == "true" ]; then echo "true"; else echo "false"; fi) \
           --set global.enableDNSNameResolver=$(if [ "${OVN_ENABLE_DNSNAMERESOLVER}" == "true" ]; then echo "true"; else echo "false"; fi) \
           ${ovnkube_db_options}
 }

contrib/kind.sh

@@ -79,6 +79,7 @@ usage() {
     echo "                 [-ic | --enable-interconnect]"
     echo "                 [--isolated]"
     echo "                 [-dns | --enable-dnsnameresolver]"
+    echo "                 [-obs | --observability]"
     echo "                 [-h]]"
     echo ""
     echo "-cf  | --config-file                Name of the KIND J2 configuration file."
@@ -141,6 +142,7 @@ usage() {
     echo "--deploy                            Deploy ovn kubernetes without restarting kind"
     echo "--add-nodes                         Adds nodes to an existing cluster. The number of nodes to be added is specified by --num-workers. Also use -ic if the cluster is using interconnect."
     echo "-dns | --enable-dnsnameresolver     Enable DNSNameResolver for resolving the DNS names used in the DNS rules of EgressFirewall."
+    echo "-obs | --observability              Enable OVN Observability feature."
     echo ""
 }
 
@@ -163,6 +165,8 @@ parse_args() {
                                                 ;;
             -ikv | --install-kubevirt)          KIND_INSTALL_KUBEVIRT=true
                                                 ;;
+            -nokvipam | --opt-out-kv-ipam)      KIND_OPT_OUT_KUBEVIRT_IPAM=true
+                                                ;;
             -ha | --ha-enabled )                OVN_HA=true
                                                 ;;
             -me | --multicast-enabled)          OVN_MULTICAST_ENABLE=true
@@ -199,6 +203,8 @@ parse_args() {
             -ifa | --ipfix-cache-active-timeout ) shift
                                                 OVN_IPFIX_CACHE_ACTIVE_TIMEOUT=$1
                                                 ;;
+            -obs | --observability )            OVN_OBSERV_ENABLE=true
+                                                ;;
             -el | --ovn-empty-lb-events )       OVN_EMPTY_LB_EVENTS=true
                                                 ;;
             -kt | --keep-taint )                KIND_REMOVE_TAINT=false
@@ -334,7 +340,8 @@ parse_args() {
             -h | --help )                       usage
                                                 exit
                                                 ;;
-            * )                                 usage
+            * )                                 echo "Invalid option: $1"
+                                                usage
                                                 exit 1
         esac
         shift
@@ -350,6 +357,7 @@ print_params() {
      echo "KIND_INSTALL_METALLB = $KIND_INSTALL_METALLB"
      echo "KIND_INSTALL_PLUGINS = $KIND_INSTALL_PLUGINS"
      echo "KIND_INSTALL_KUBEVIRT = $KIND_INSTALL_KUBEVIRT"
+     echo "KIND_OPT_OUT_KUBEVIRT_IPAM = $KIND_OPT_OUT_KUBEVIRT_IPAM"
      echo "OVN_HA = $OVN_HA"
      echo "RUN_IN_CONTAINER = $RUN_IN_CONTAINER"
      echo "KIND_CLUSTER_NAME = $KIND_CLUSTER_NAME"
@@ -377,6 +385,7 @@ print_params() {
      echo "OVN_IPFIX_SAMPLING = $OVN_IPFIX_SAMPLING"
      echo "OVN_IPFIX_CACHE_MAX_FLOWS = $OVN_IPFIX_CACHE_MAX_FLOWS"
      echo "OVN_IPFIX_CACHE_ACTIVE_TIMEOUT = $OVN_IPFIX_CACHE_ACTIVE_TIMEOUT"
+     echo "OVN_OBSERV_ENABLE = $OVN_OBSERV_ENABLE"
      echo "OVN_EMPTY_LB_EVENTS = $OVN_EMPTY_LB_EVENTS"
      echo "OVN_MULTICAST_ENABLE = $OVN_MULTICAST_ENABLE"
      echo "OVN_IMAGE = $OVN_IMAGE"
@@ -503,6 +512,7 @@ set_default_params() {
   KIND_INSTALL_METALLB=${KIND_INSTALL_METALLB:-false}
   KIND_INSTALL_PLUGINS=${KIND_INSTALL_PLUGINS:-false}
   KIND_INSTALL_KUBEVIRT=${KIND_INSTALL_KUBEVIRT:-false}
+  KIND_OPT_OUT_KUBEVIRT_IPAM=${KIND_OPT_OUT_KUBEVIRT_IPAM:-false}
   OVN_HA=${OVN_HA:-false}
   KIND_LOCAL_REGISTRY=${KIND_LOCAL_REGISTRY:-false}
   KIND_LOCAL_REGISTRY_NAME=${KIND_LOCAL_REGISTRY_NAME:-kind-registry}
@@ -604,6 +614,7 @@ set_default_params() {
   fi
   OVN_MTU=${OVN_MTU:-1400}
   OVN_ENABLE_DNSNAMERESOLVER=${OVN_ENABLE_DNSNAMERESOLVER:-false}
+  OVN_OBSERV_ENABLE=${OVN_OBSERV_ENABLE:-false}
 }
 
 check_ipv6() {
@@ -851,7 +862,9 @@ create_ovn_kube_manifests() {
     --enable-ovnkube-identity="${OVN_ENABLE_OVNKUBE_IDENTITY}" \
     --enable-persistent-ips=true \
     --mtu="${OVN_MTU}" \
-    --enable-dnsnameresolver="${OVN_ENABLE_DNSNAMERESOLVER}"
+    --enable-dnsnameresolver="${OVN_ENABLE_DNSNAMERESOLVER}" \
+    --mtu="${OVN_MTU}" \
+    --enable-observ="${OVN_OBSERV_ENABLE}"
   popd
 }
 
@@ -1163,5 +1176,11 @@ if [ "$KIND_INSTALL_PLUGINS" == true ]; then
 fi
 if [ "$KIND_INSTALL_KUBEVIRT" == true ]; then
   install_kubevirt
-  install_kubevirt_ipam_controller
+  deploy_kubevirt_binding
+  deploy_passt_binary
+
+  install_cert_manager
+  if [ "$KIND_OPT_OUT_KUBEVIRT_IPAM" != true ]; then
+    install_kubevirt_ipam_controller
+  fi
 fi

dist/images/Dockerfile.fedora

@@ -9,13 +9,13 @@
 # are built locally and included in the image (instead of the rpm)
 #
 
-FROM fedora:rawhide
+FROM fedora:41
 
 USER root
 
 ENV PYTHONDONTWRITEBYTECODE yes
 
-ARG ovnver=ovn-24.03.90-6.fc42
+ARG ovnver=ovn-24.03.90-7.fc41
 # Automatically populated when using docker buildx
 ARG TARGETPLATFORM
 ARG BUILDPLATFORM
@@ -37,14 +37,14 @@ RUN ln -s /usr/bin/python3 /usr/libexec/platform-python
 RUN mkdir -p /var/run/openvswitch
 
 RUN if [ "$TARGETPLATFORM" = "linux/amd64" ] || [ -z "$TARGETPLATFORM"] ; then koji download-build $ovnver --arch=x86_64  ; \
-    else koji download-build $ovnver --arch=aarch64 ; fi 
+    else koji download-build $ovnver --arch=aarch64 ; fi
 
 RUN rpm -Uhv --nodeps --force *.rpm
 
 # Built in ../../go_controller, then the binaries are copied here.
 # put things where they are in the pkg
 RUN mkdir -p /usr/libexec/cni/
-COPY ovnkube ovn-kube-util ovndbchecker hybrid-overlay-node ovnkube-identity /usr/bin/
+COPY ovnkube ovn-kube-util ovndbchecker hybrid-overlay-node ovnkube-identity ovnkube-observ /usr/bin/
 COPY ovn-k8s-cni-overlay /usr/libexec/cni/ovn-k8s-cni-overlay
 
 # ovnkube.sh is the entry point. This script examines environment

dist/images/Dockerfile.fedora.dev

@@ -83,6 +83,7 @@ COPY ovn-kube-util /usr/bin/
 COPY ovndbchecker /usr/bin/
 COPY hybrid-overlay-node /usr/bin
 COPY ovnkube-identity /usr/bin/
+COPY ovnkube-observ /usr/bin
 COPY ovn-k8s-cni-overlay /usr/libexec/cni/ovn-k8s-cni-overlay
 
 # ovnkube.sh is the entry point. This script examines environment

dist/images/Dockerfile.ubuntu

@@ -27,7 +27,7 @@ RUN mkdir -p /var/run/openvswitch
 # Built in ../../go_controller, then the binaries are copied here.
 # put things where they are in the pkg
 RUN mkdir -p /usr/libexec/cni/
-COPY ovnkube ovn-kube-util ovndbchecker hybrid-overlay-node ovnkube-identity /usr/bin/
+COPY ovnkube ovn-kube-util ovndbchecker hybrid-overlay-node ovnkube-identity ovnkube-observ /usr/bin/
 COPY ovn-k8s-cni-overlay /usr/libexec/cni/ovn-k8s-cni-overlay
 
 # ovnkube.sh is the entry point. This script examines environment

dist/images/daemonset.sh

@@ -99,6 +99,7 @@ OVN_ENABLE_DNSNAMERESOLVER="false"
 IN_UPGRADE=
 # northd-backoff-interval, in ms
 OVN_NORTHD_BACKOFF_INTERVAL=
+OVN_OBSERV_ENABLE="false"
 
 # Parse parameters given as arguments to this script.
 while [ "$1" != "" ]; do
@@ -354,6 +355,9 @@ while [ "$1" != "" ]; do
   --enable-dnsnameresolver)
     OVN_ENABLE_DNSNAMERESOLVER=$VALUE
     ;;
+  --enable-observ)
+    OVN_OBSERV_ENABLE=$VALUE
+    ;;
   *)
     echo "WARNING: unknown parameter \"$PARAM\""
     exit 1
@@ -544,6 +548,9 @@ echo "ovn_enable_svc_template_support: ${ovn_enable_svc_template_support}"
 ovn_enable_dnsnameresolver=${OVN_ENABLE_DNSNAMERESOLVER}
 echo "ovn_enable_dnsnameresolver: ${ovn_enable_dnsnameresolver}"
 
+ovn_observ_enable=${OVN_OBSERV_ENABLE}
+echo "ovn_observ_enable: ${ovn_observ_enable}"
+
 ovn_image=${ovnkube_image} \
   ovnkube_compact_mode_enable=${ovnkube_compact_mode_enable} \
   ovn_image_pull_policy=${image_pull_policy} \
@@ -592,6 +599,7 @@ ovn_image=${ovnkube_image} \
   ovn_enable_interconnect=${ovn_enable_interconnect} \
   ovn_enable_multi_external_gateway=${ovn_enable_multi_external_gateway} \
   ovn_enable_ovnkube_identity=${ovn_enable_ovnkube_identity} \
+  ovn_observ_enable=${ovn_observ_enable} \
   ovnkube_app_name=ovnkube-node \
   jinjanate ../templates/ovnkube-node.yaml.j2 -o ${output_dir}/ovnkube-node.yaml
 
@@ -643,6 +651,7 @@ ovn_image=${ovnkube_image} \
   ovn_enable_interconnect=${ovn_enable_interconnect} \
   ovn_enable_multi_external_gateway=${ovn_enable_multi_external_gateway} \
   ovn_enable_ovnkube_identity=${ovn_enable_ovnkube_identity} \
+  ovn_observ_enable=${ovn_observ_enable} \
   ovnkube_app_name=ovnkube-node-dpu \
   jinjanate ../templates/ovnkube-node.yaml.j2 -o ${output_dir}/ovnkube-node-dpu.yaml
 
@@ -733,6 +742,7 @@ ovn_image=${ovnkube_image} \
   ovn_enable_persistent_ips=${ovn_enable_persistent_ips} \
   ovn_enable_svc_template_support=${ovn_enable_svc_template_support} \
   ovn_enable_dnsnameresolver=${ovn_enable_dnsnameresolver} \
+  ovn_observ_enable=${ovn_observ_enable} \
   jinjanate ../templates/ovnkube-master.yaml.j2 -o ${output_dir}/ovnkube-master.yaml
 
 ovn_image=${ovnkube_image} \
@@ -774,6 +784,7 @@ ovn_image=${ovnkube_image} \
   ovn_v6_transit_switch_subnet=${ovn_v6_transit_switch_subnet} \
   ovn_enable_persistent_ips=${ovn_enable_persistent_ips} \
   ovn_enable_dnsnameresolver=${ovn_enable_dnsnameresolver} \
+  ovn_observ_enable=${ovn_observ_enable} \
   jinjanate ../templates/ovnkube-control-plane.yaml.j2 -o ${output_dir}/ovnkube-control-plane.yaml
 
 ovn_image=${image} \
@@ -869,6 +880,7 @@ ovn_image=${ovnkube_image} \
   ovn_enable_persistent_ips=${ovn_enable_persistent_ips} \
   ovn_enable_svc_template_support=${ovn_enable_svc_template_support} \
   ovn_enable_dnsnameresolver=${ovn_enable_dnsnameresolver} \
+  ovn_observ_enable=${ovn_observ_enable} \
   jinjanate ../templates/ovnkube-single-node-zone.yaml.j2 -o ${output_dir}/ovnkube-single-node-zone.yaml
 
 ovn_image=${ovnkube_image} \
@@ -932,6 +944,7 @@ ovn_image=${ovnkube_image} \
   ovn_enable_persistent_ips=${ovn_enable_persistent_ips} \
   ovn_enable_svc_template_support=${ovn_enable_svc_template_support} \
   ovn_enable_dnsnameresolver=${ovn_enable_dnsnameresolver} \
+  ovn_observ_enable=${ovn_observ_enable} \
   jinjanate ../templates/ovnkube-zone-controller.yaml.j2 -o ${output_dir}/ovnkube-zone-controller.yaml
 
 ovn_image=${image} \