Merge pull request #1718 from shiftstack/gce-image

🌱 ci: Several CI fixes

hack/ci/create_devstack.sh

@@ -77,32 +77,26 @@ function retry {
 function ensure_openstack_client {
     if ! command -v openstack;
     then
-        # We are running in a Debian Buster image with python 3.7. Python 3.7
-        # is starting to show its age in upstream support. Ideally we would
-        # move to a newer image with a newer python version.
-        #
-        # Until then, this script tries to carefully navigate around current
-        # issues running openstack client on python 3.7.
-        #
-        # We explicitly pin the yoga version of openstackclient. This is the
-        # last version of openstackclient which will support python 3.7.
-
         # Install virtualenv to install the openstack client and curl to fetch
         # the build constraints.
         apt-get install -y python3-virtualenv curl
         python3 -m virtualenv -p $(which python3) /tmp/openstack-venv
         VIRTUAL_ENV_DISABLE_PROMPT=1 source /tmp/openstack-venv/bin/activate
 
-        # openstackclient has never actually supported python 3.7, only 3.6 and
-        # 3.8. Here we download the yoga constraints file and modify all the
-        # 3.8 constraints to be 3.7 constraints.
-        curl -L https://releases.openstack.org/constraints/upper/yoga -o /tmp/yoga-constraints
-        sed -i "s/python_version=='3.8'/python_version=='3.7'/" /tmp/yoga-constraints
+        # We explicitly pin to the stable branch version of openstackclient.
+        curl -L https://releases.openstack.org/constraints/upper/${OPENSTACK_RELEASE} -o /tmp/openstack-constraints
+
+        # Hack for yoga only: wrapt <1.14 doesn't support python 3.11
+        [ "${OPENSTACK_RELEASE}" == "yoga" ] || exit 1 # Delete this hack
+        sed -i "s/^wrapt===1\.13.*/wrapt===1.14.1/" /tmp/openstack-constraints
 
-        pip install -c /tmp/yoga-constraints \
+        pip install -c /tmp/openstack-constraints \
                 python-openstackclient python-cinderclient \
                 python-glanceclient python-keystoneclient \
                 python-neutronclient python-novaclient python-octaviaclient
+
+        # Ensure openstack cli can load fully including all plugins
+        openstack help >/dev/null
     fi
 }
 

hack/ci/gce-project.sh

@@ -85,37 +85,17 @@ function create_vm {
   local machine_type="GCP_MACHINE_TYPE_${name}"
   machine_type=${!machine_type}
   local servername="${CLUSTER_NAME}-${name}"
-  local diskname="${CLUSTER_NAME}-disk"
-  local imagename="${servername}-image"
 
   # Loop over all zones in the GCP region to ignore a full zone.
   # We are not able to use 'gcloud compute zones list' as the gcloud.compute.zones.list permission is missing.
   for GCP_ZONE in "${GCP_REGION}-a" "${GCP_REGION}-b" "${GCP_REGION}-c"; do
-    # Check if image was already created.
-    # Images are not zone specific, but the disk is.
-    if ! gcloud compute images describe "$imagename" --project "$GCP_PROJECT" >/dev/null; then
-      # Create the base disk image based on the public Ubuntu 22.04 LTS cloud image
-      # Note that this has also been verified to work with CentOS 8 as of
-      # 2021-01-12, but this is not tested regularly.
-      # To use CentOS 8:
-      #   --image-project centos-cloud --image-family centos-stream-8
-      if ! gcloud compute disks describe "$diskname" --project "$GCP_PROJECT" --zone "$GCP_ZONE" >/dev/null; then
-        gcloud compute disks create "$diskname" \
-          --project "$GCP_PROJECT" \
-          --image-project ubuntu-os-cloud --image-family ubuntu-2204-lts \
-          --zone "$GCP_ZONE"
-      fi
-      gcloud compute images create "$imagename" \
-        --project "$GCP_PROJECT" \
-        --source-disk "$diskname" --source-disk-zone "$GCP_ZONE" \
-        --licenses "https://www.googleapis.com/compute/v1/projects/vm-options/global/licenses/enable-vmx"
-    fi
-
     if ! gcloud compute instances describe "$servername" --project "$GCP_PROJECT" --zone "$GCP_ZONE" >/dev/null; then
       if gcloud compute instances create "$servername" \
         --project "$GCP_PROJECT" \
         --zone "$GCP_ZONE" \
-        --image "$imagename" \
+        --enable-nested-virtualization \
+        --image-project ubuntu-os-cloud \
+        --image-family ubuntu-2004-lts \
         --boot-disk-size 200G \
         --boot-disk-type pd-ssd \
         --can-ip-forward \

hack/ensure-kubectl.sh

@@ -40,7 +40,7 @@ verify_kubectl_version() {
   fi
 
   local kubectl_version
-  IFS=" " read -ra kubectl_version <<< "$(kubectl version --client --short)"
+  IFS=" " read -ra kubectl_version <<< "$(kubectl version --client)"
   if [[ "${MINIMUM_KUBECTL_VERSION}" != $(echo -e "${MINIMUM_KUBECTL_VERSION}\n${kubectl_version[2]}" | sort -s -t. -k 1,1 -k 2,2n -k 3,3n | head -n1) ]]; then
     cat <<EOF
 Detected kubectl version: ${kubectl_version[2]}.

scripts/ci-conformance.sh

@@ -45,14 +45,12 @@ cleanup() {
 }
 trap cleanup EXIT
 
-# Ensure that python3-pip is installed.
 apt-get update -y
-apt-get install -y python3-pip
+# Install requests module explicitly for HTTP calls.
+# libffi required for pip install cffi (yoga dependency)
+apt-get install -y python3-requests libffi-dev
 rm -rf /var/lib/apt/lists/*
 
-# Install/upgrade pip and requests module explicitly for HTTP calls.
-python3 -m pip install --upgrade pip requests
-
 # If BOSKOS_HOST is set then acquire a resource of type ${RESOURCE_TYPE} from Boskos.
 if [ -n "${BOSKOS_HOST:-}" ]; then
   # Check out the account from Boskos and store the produced environment

scripts/ci-e2e.sh

@@ -46,14 +46,12 @@ cleanup() {
 }
 trap cleanup EXIT
 
-# Ensure that python3-pip is installed.
 apt-get update -y
-apt-get install -y python3-pip
+# Install requests module explicitly for HTTP calls.
+# libffi required for pip install cffi (yoga dependency)
+apt-get install -y python3-requests libffi-dev
 rm -rf /var/lib/apt/lists/*
 
-# Install/upgrade pip and requests module explicitly for HTTP calls.
-python3 -m pip install --upgrade pip requests
-
 # If BOSKOS_HOST is set then acquire a resource of type ${RESOURCE_TYPE} from Boskos.
 if [ -n "${BOSKOS_HOST:-}" ]; then
   # Check out the account from Boskos and store the produced environment