Merge pull request #564 from derekhiggins/download-containers

Move image downloads into resource downloader containers

01_install_requirements.sh

@@ -63,7 +63,6 @@ sudo yum -y install \
   ansible \
   bind-utils \
   jq \
-  libguestfs-tools \
   libvirt \
   libvirt-devel \
   libvirt-daemon-kvm \

04_setup_ironic.sh

@@ -5,13 +5,10 @@ set -ex
 source logging.sh
 source common.sh
 
-# Get the various images
-source get_images.sh
-
 # Either pull or build the ironic images
 # To build the IRONIC image set
 # IRONIC_IMAGE=https://github.com/metalkube/metalkube-ironic
-for IMAGE_VAR in IRONIC_IMAGE IRONIC_INSPECTOR_IMAGE ; do
+for IMAGE_VAR in IRONIC_IMAGE IRONIC_INSPECTOR_IMAGE IPA_DOWNLOADER_IMAGE COREOS_DOWNLOADER_IMAGE ; do
     IMAGE=${!IMAGE_VAR}
     # Is it a git repo?
     if [[ "$IMAGE" =~ "://" ]] ; then
@@ -27,24 +24,7 @@ for IMAGE_VAR in IRONIC_IMAGE IRONIC_INSPECTOR_IMAGE ; do
     fi
 done
 
-pushd $IRONIC_DATA_DIR/html/images
-
-# Compress the qcow2 image so that it can be downloaded into
-# a smaller /tmp by IPA
-if [ ! -e "$RHCOS_IMAGE_FILENAME_COMPRESSED" ] ; then
-    qemu-img convert -O qcow2 -c "$RHCOS_IMAGE_FILENAME_OPENSTACK" "$RHCOS_IMAGE_FILENAME_COMPRESSED"
-fi
-
-if [ ! -e "${RHCOS_IMAGE_FILENAME_COMPRESSED}.md5sum" -o \
-     "$RHCOS_IMAGE_FILENAME_COMPRESSED" -nt "$RHCOS_IMAGE_FILENAME_COMPRESSED.md5sum" ] ; then
-    md5sum "$RHCOS_IMAGE_FILENAME_COMPRESSED" | cut -f 1 -d " " > "$RHCOS_IMAGE_FILENAME_COMPRESSED.md5sum"
-fi
-
-ln -sf "$RHCOS_IMAGE_FILENAME_COMPRESSED" "$RHCOS_IMAGE_FILENAME_LATEST"
-ln -sf "$RHCOS_IMAGE_FILENAME_COMPRESSED.md5sum" "$RHCOS_IMAGE_FILENAME_LATEST.md5sum"
-popd
-
-for name in ironic ironic-inspector dnsmasq httpd mariadb; do
+for name in ironic ironic-inspector dnsmasq httpd mariadb ipa-downloader coreos-downloader; do
     sudo podman ps | grep -w "$name$" && sudo podman kill $name
     sudo podman ps --all | grep -w "$name$" && sudo podman rm $name -f
 done
@@ -76,6 +56,15 @@ sudo podman run -d --net host --privileged --name ironic --pod ironic-pod \
      --env OS_CONDUCTOR__HEARTBEAT_TIMEOUT=120 \
      -v $IRONIC_DATA_DIR:/shared ${IRONIC_IMAGE}
 
+sudo podman run -d --net host --privileged --name ipa-downloader --pod ironic-pod \
+     -v $IRONIC_DATA_DIR:/shared ${IPA_DOWNLOADER_IMAGE} /usr/local/bin/get-resource.sh
+
+sudo podman run -d --net host --privileged --name coreos-downloader --pod ironic-pod \
+     -v $IRONIC_DATA_DIR:/shared ${COREOS_DOWNLOADER_IMAGE} /usr/local/bin/get-resource.sh $RHCOS_IMAGE_URL
+
 # Start Ironic Inspector 
 sudo podman run -d --net host --privileged --name ironic-inspector \
      --pod ironic-pod -v $IRONIC_DATA_DIR:/shared "${IRONIC_INSPECTOR_IMAGE}"
+
+# Wait for images to be downloaded/ready
+while ! curl --fail http://localhost:80/images/rhcos-ootpa-latest.qcow2.md5sum ; do sleep 1 ; done

common.sh

@@ -49,19 +49,13 @@ export VM_EXTRADISKS=${VM_EXTRADISKS:-"false"}
 
 export RHCOS_INSTALLER_IMAGE_URL="https://releases-art-rhcos.svc.ci.openshift.org/art/storage/releases/ootpa/410.8.20190520.0/"
 export RHCOS_IMAGE_URL=${RHCOS_IMAGE_URL:-${RHCOS_INSTALLER_IMAGE_URL}}
-
-export RHCOS_IMAGE_FILENAME_OPENSTACK_GZ="$(curl ${RHCOS_IMAGE_URL}/meta.json | jq -r '.images.openstack.path')"
-export RHCOS_IMAGE_NAME=$(echo $RHCOS_IMAGE_FILENAME_OPENSTACK_GZ | sed -e 's/-openstack.*//')
-# FIXME(shardy) - we need to download the -openstack as its needed
-# for the baremetal nodes so we get config drive support,
-# or perhaps a completely new image?
-export RHCOS_IMAGE_FILENAME_OPENSTACK="${RHCOS_IMAGE_NAME}-openstack.qcow2"
-export RHCOS_IMAGE_FILENAME_COMPRESSED="${RHCOS_IMAGE_NAME}-compressed.qcow2"
 export RHCOS_IMAGE_FILENAME_LATEST="rhcos-ootpa-latest.qcow2"
 
 # Ironic vars
 export IRONIC_IMAGE=${IRONIC_IMAGE:-"quay.io/metal3-io/ironic:master"}
 export IRONIC_INSPECTOR_IMAGE=${IRONIC_INSPECTOR_IMAGE:-"quay.io/metal3-io/ironic-inspector:master"}
+export IPA_DOWNLOADER_IMAGE=${IPA_DOWNLOADER_IMAGE:-"quay.io/metal3-io/ironic-ipa-downloader:master"}
+export COREOS_DOWNLOADER_IMAGE=${COREOS_DOWNLOADER_IMAGE:-"quay.io/openshift-metal3/rhcos-downloader:master"}
 export IRONIC_DATA_DIR="$WORKING_DIR/ironic"
 
 export KUBECONFIG="${SCRIPTDIR}/ocp/auth/kubeconfig"

run_ci.sh

@@ -11,6 +11,10 @@ function getlogs(){
     # The logs shared by the ironic containers
     sudo cp -r /opt/dev-scripts/ironic/log $LOGDIR/container-logs
 
+
+    sudo podman logs coreos-downloader > $LOGDIR/coreos-downloader.log
+    sudo podman logs ipa-downloader > $LOGDIR/ipa-downloader.log
+
     # And the VM jornals
     for HOST in $(sudo virsh net-dhcp-leases baremetal | grep -o '192.168.111.[0-9]\+') ; do
         sshpass -p notworking $SSH core@$HOST sudo journalctl > $LOGDIR/$HOST-system.journal || true
@@ -50,16 +54,6 @@ mount | grep root-
 FILECACHEDIR=/opt/data/filecache
 FILESTOCACHE="/opt/dev-scripts/ironic/html/images/ironic-python-agent.initramfs /opt/dev-scripts/ironic/html/images/ironic-python-agent.kernel"
 
-# Check if we have any openstack images cached, and if so add the most
-# recent to FILESTOCACHE
-compgen -G "$FILECACHEDIR/*-openstack.qcow2"
-retval=$?
-if [ $retval -eq 0 ]
-then
-  LAST_OPENSTACK_IMAGE=$(ls -r $FILECACHEDIR/*-openstack.qcow2 | head -n1)
-  FILESTOCACHE="$FILESTOCACHE /opt/dev-scripts/ironic/html/images/$(basename $LAST_OPENSTACK_IMAGE)"
-fi
-
 # Because "/" is a btrfs subvolume snapshot and a new one is created for each CI job
 # to prevent each snapshot taking up too much space we keep some of the larger files
 # on /opt we need to delete these before the job starts
@@ -132,9 +126,6 @@ done
 set -o pipefail
 timeout -s 9 85m make |& ts "%b %d %H:%M:%S | " |& sed -e 's/.*auths.*/*** PULL_SECRET ***/g'
 
-source common.sh
-FILESTOCACHE="$FILESTOCACHE /opt/dev-scripts/ironic/html/images/$RHCOS_IMAGE_FILENAME_OPENSTACK"
-
 # Populate cache for files it doesn't have, or that have changed
 for FILE in $FILESTOCACHE ; do
     cached=$FILECACHEDIR/$(basename $FILE)