Bug 1772599: Handle compressed images for libvirt and baremetal IPI

pkg/rhcos/openstack.go

@@ -3,7 +3,6 @@ package rhcos
 import (
 	"context"
 	"net/url"
-	"strings"
 
 	"github.com/pkg/errors"
 )
@@ -28,14 +27,10 @@ func OpenStack(ctx context.Context) (string, error) {
 
 	baseURL := base.ResolveReference(relOpenStack).String()
 
-	// Attach sha256 checksum to the URL. If the file has the ".gz" extension, then the
-	// data is compressed and we use SHA256 value; otherwise we work with uncompressed
-	// data and therefore need UncompressedSHA256.
-	if strings.HasSuffix(baseURL, ".gz") {
-		baseURL += "?sha256=" + meta.Images.OpenStack.SHA256
-	} else {
-		baseURL += "?sha256=" + meta.Images.OpenStack.UncompressedSHA256
-	}
+	// Attach sha256 checksum to the URL.  Always provide the
+	// uncompressed SHA256; the cache will take care of
+	// uncompressing before checksumming.
+	baseURL += "?sha256=" + meta.Images.OpenStack.UncompressedSHA256
 
 	// Check that we have generated a valid URL
 	_, err = url.ParseRequestURI(baseURL)

pkg/rhcos/qemu.go

@@ -24,5 +24,18 @@ func QEMU(ctx context.Context) (string, error) {
 		return "", err
 	}
 
-	return base.ResolveReference(relQEMU).String(), nil
+	baseURL := base.ResolveReference(relQEMU).String()
+
+	// Attach sha256 checksum to the URL.  Always provide the
+	// uncompressed SHA256; the cache will take care of
+	// uncompressing before checksumming.
+	baseURL += "?sha256=" + meta.Images.QEMU.UncompressedSHA256
+
+	// Check that we have generated a valid URL
+	_, err = url.ParseRequestURI(baseURL)
+	if err != nil {
+		return "", err
+	}
+
+	return baseURL, nil
 }

pkg/tfvars/baremetal/baremetal.go

@@ -6,7 +6,7 @@ import (
 	"fmt"
 	"github.com/metal3-io/baremetal-operator/pkg/bmc"
 	"github.com/metal3-io/baremetal-operator/pkg/hardware"
-	libvirttfvars "github.com/openshift/installer/pkg/tfvars/libvirt"
+	"github.com/openshift/installer/pkg/tfvars/internal/cache"
 	"github.com/openshift/installer/pkg/types/baremetal"
 	"github.com/pkg/errors"
 	"net/url"
@@ -31,7 +31,7 @@ type config struct {
 
 // TFVars generates bare metal specific Terraform variables.
 func TFVars(libvirtURI, bootstrapProvisioningIP, bootstrapOSImage, externalBridge, provisioningBridge string, platformHosts []*baremetal.Host, image string) ([]byte, error) {
-	bootstrapOSImage, err := libvirttfvars.CachedImage(bootstrapOSImage)
+	bootstrapOSImage, err := cache.DownloadImageFile(bootstrapOSImage)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to use cached bootstrap libvirt image")
 	}

pkg/tfvars/internal/cache/cache.go

@@ -1,6 +1,8 @@
 package cache
 
 import (
+	"bytes"
+	"compress/gzip"
 	"crypto/md5"
 	"crypto/sha256"
 	"fmt"
@@ -19,6 +21,7 @@ import (
 const (
 	applicationName = "openshift-installer"
 	imageDataType   = "image"
+	gzipFileType    = "application/x-gzip"
 )
 
 // getCacheDir returns a local path of the cache, where the installer should put the data:
@@ -107,6 +110,29 @@ func cacheFile(reader io.Reader, filePath string, sha256Checksum string) (err er
 		}
 	}()
 
+	// Detect whether we know how to decompress the file
+	// See http://golang.org/pkg/net/http/#DetectContentType for why we use 512
+	buf := make([]byte, 512)
+	_, err = reader.Read(buf)
+	if err != nil {
+		return err
+	}
+
+	reader = io.MultiReader(bytes.NewReader(buf), reader)
+	fileType := http.DetectContentType(buf)
+	logrus.Debugf("content type of %s is %s", filePath, fileType)
+	switch fileType {
+	case gzipFileType:
+		uncompressor, err := gzip.NewReader(reader)
+		if err != nil {
+			return err
+		}
+		defer uncompressor.Close()
+		reader = uncompressor
+	default:
+		// No need for an interposer otherwise
+	}
+
 	// Wrap the reader in TeeReader to calculate sha256 checksum on the fly
 	hasher := sha256.New()
 	if sha256Checksum != "" {
@@ -193,7 +219,8 @@ func DownloadFile(baseURL string, dataType string) (string, error) {
 }
 
 // DownloadImageFile is a helper function that obtains an image file from a given URL,
-// puts it in the cache and returns the local file path.
+// puts it in the cache and returns the local file path.  If the file is compressed
+// by a known compressor, the file is uncompressed prior to being returned.
 func DownloadImageFile(baseURL string) (string, error) {
 	logrus.Infof("Obtaining RHCOS image file from '%v'", baseURL)
 

pkg/tfvars/libvirt/libvirt.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/apparentlymart/go-cidr/cidr"
 	"github.com/openshift/cluster-api-provider-libvirt/pkg/apis/libvirtproviderconfig/v1beta1"
+	"github.com/openshift/installer/pkg/tfvars/internal/cache"
 	"github.com/pkg/errors"
 )
 
@@ -34,7 +35,7 @@ func TFVars(masterConfig *v1beta1.LibvirtMachineProviderConfig, osImage string,
 		return nil, err
 	}
 
-	osImage, err = cachedImage(osImage)
+	osImage, err = cache.DownloadImageFile(osImage)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to use cached libvirt image")
 	}