From d9955c20d5707a11c41930c4df42936d5cf811ac Mon Sep 17 00:00:00 2001 From: Robert Krawitz Date: Tue, 12 Nov 2019 16:41:33 -0500 Subject: [PATCH] Cache uncompressed images, as installers can't actually use compressed images. --- pkg/rhcos/openstack.go | 13 +-- pkg/rhcos/qemu.go | 15 ++- pkg/tfvars/baremetal/baremetal.go | 4 +- pkg/tfvars/internal/cache/cache.go | 29 ++++- pkg/tfvars/libvirt/cache.go | 176 ----------------------------- pkg/tfvars/libvirt/libvirt.go | 3 +- pkg/tfvars/openstack/openstack.go | 60 +--------- 7 files changed, 51 insertions(+), 249 deletions(-) delete mode 100644 pkg/tfvars/libvirt/cache.go diff --git a/pkg/rhcos/openstack.go b/pkg/rhcos/openstack.go index 98bd8640e7b..d5485ef40bf 100644 --- a/pkg/rhcos/openstack.go +++ b/pkg/rhcos/openstack.go @@ -3,7 +3,6 @@ package rhcos import ( "context" "net/url" - "strings" "github.com/pkg/errors" ) @@ -28,14 +27,10 @@ func OpenStack(ctx context.Context) (string, error) { baseURL := base.ResolveReference(relOpenStack).String() - // Attach sha256 checksum to the URL. If the file has the ".gz" extension, then the - // data is compressed and we use SHA256 value; otherwise we work with uncompressed - // data and therefore need UncompressedSHA256. - if strings.HasSuffix(baseURL, ".gz") { - baseURL += "?sha256=" + meta.Images.OpenStack.SHA256 - } else { - baseURL += "?sha256=" + meta.Images.OpenStack.UncompressedSHA256 - } + // Attach sha256 checksum to the URL. Always provide the + // uncompressed SHA256; the cache will take care of + // uncompressing before checksumming. + baseURL += "?sha256=" + meta.Images.OpenStack.UncompressedSHA256 // Check that we have generated a valid URL _, err = url.ParseRequestURI(baseURL) diff --git a/pkg/rhcos/qemu.go b/pkg/rhcos/qemu.go index e4408447cfa..5ada808d48c 100644 --- a/pkg/rhcos/qemu.go +++ b/pkg/rhcos/qemu.go @@ -24,5 +24,18 @@ func QEMU(ctx context.Context) (string, error) { return "", err } - return base.ResolveReference(relQEMU).String(), nil + baseURL := base.ResolveReference(relQEMU).String() + + // Attach sha256 checksum to the URL. Always provide the + // uncompressed SHA256; the cache will take care of + // uncompressing before checksumming. + baseURL += "?sha256=" + meta.Images.QEMU.UncompressedSHA256 + + // Check that we have generated a valid URL + _, err = url.ParseRequestURI(baseURL) + if err != nil { + return "", err + } + + return baseURL, nil } diff --git a/pkg/tfvars/baremetal/baremetal.go b/pkg/tfvars/baremetal/baremetal.go index 1b070a064df..768388fbb9c 100644 --- a/pkg/tfvars/baremetal/baremetal.go +++ b/pkg/tfvars/baremetal/baremetal.go @@ -6,7 +6,7 @@ import ( "fmt" "github.com/metal3-io/baremetal-operator/pkg/bmc" "github.com/metal3-io/baremetal-operator/pkg/hardware" - libvirttfvars "github.com/openshift/installer/pkg/tfvars/libvirt" + "github.com/openshift/installer/pkg/tfvars/internal/cache" "github.com/openshift/installer/pkg/types/baremetal" "github.com/pkg/errors" "net/url" @@ -31,7 +31,7 @@ type config struct { // TFVars generates bare metal specific Terraform variables. func TFVars(libvirtURI, bootstrapProvisioningIP, bootstrapOSImage, externalBridge, provisioningBridge string, platformHosts []*baremetal.Host, image string) ([]byte, error) { - bootstrapOSImage, err := libvirttfvars.CachedImage(bootstrapOSImage) + bootstrapOSImage, err := cache.DownloadImageFile(bootstrapOSImage) if err != nil { return nil, errors.Wrap(err, "failed to use cached bootstrap libvirt image") } diff --git a/pkg/tfvars/internal/cache/cache.go b/pkg/tfvars/internal/cache/cache.go index 126fb5300ce..64ef6e97d3a 100644 --- a/pkg/tfvars/internal/cache/cache.go +++ b/pkg/tfvars/internal/cache/cache.go @@ -1,6 +1,8 @@ package cache import ( + "bytes" + "compress/gzip" "crypto/md5" "crypto/sha256" "fmt" @@ -19,6 +21,7 @@ import ( const ( applicationName = "openshift-installer" imageDataType = "image" + gzipFileType = "application/x-gzip" ) // getCacheDir returns a local path of the cache, where the installer should put the data: @@ -107,6 +110,29 @@ func cacheFile(reader io.Reader, filePath string, sha256Checksum string) (err er } }() + // Detect whether we know how to decompress the file + // See http://golang.org/pkg/net/http/#DetectContentType for why we use 512 + buf := make([]byte, 512) + _, err = reader.Read(buf) + if err != nil { + return err + } + + reader = io.MultiReader(bytes.NewReader(buf), reader) + fileType := http.DetectContentType(buf) + logrus.Debugf("content type of %s is %s", filePath, fileType) + switch fileType { + case gzipFileType: + uncompressor, err := gzip.NewReader(reader) + if err != nil { + return err + } + defer uncompressor.Close() + reader = uncompressor + default: + // No need for an interposer otherwise + } + // Wrap the reader in TeeReader to calculate sha256 checksum on the fly hasher := sha256.New() if sha256Checksum != "" { @@ -193,7 +219,8 @@ func DownloadFile(baseURL string, dataType string) (string, error) { } // DownloadImageFile is a helper function that obtains an image file from a given URL, -// puts it in the cache and returns the local file path. +// puts it in the cache and returns the local file path. If the file is compressed +// by a known compressor, the file is uncompressed prior to being returned. func DownloadImageFile(baseURL string) (string, error) { logrus.Infof("Obtaining RHCOS image file from '%v'", baseURL) diff --git a/pkg/tfvars/libvirt/cache.go b/pkg/tfvars/libvirt/cache.go deleted file mode 100644 index 5ed46b57c01..00000000000 --- a/pkg/tfvars/libvirt/cache.go +++ /dev/null @@ -1,176 +0,0 @@ -package libvirt - -import ( - "crypto/md5" - "encoding/hex" - "fmt" - "io" - "net/http" - "os" - "path/filepath" - "strings" - - "github.com/sirupsen/logrus" - "golang.org/x/sys/unix" -) - -// CachedImage returns the location of the cached image. -// FIXME: Exported for use by baremetal platform. -func CachedImage(uri string) (string, error) { - return cachedImage(uri) -} - -// cachedImage leaves non-file:// image URIs unalterered. -// Other URIs are retrieved with a local cache at -// $XDG_CACHE_HOME/openshift-install/libvirt [1]. This allows you to -// use the same remote image URI multiple times without needing to -// worry about redundant downloads, although you will want to -// periodically blow away your cache. -// -// [1]: https://standards.freedesktop.org/basedir-spec/basedir-spec-0.7.html -func cachedImage(uri string) (string, error) { - if strings.HasPrefix(uri, "file://") { - return uri, nil - } - - logrus.Infof("Fetching OS image: %s", filepath.Base(uri)) - - baseCacheDir, err := os.UserCacheDir() - if err != nil { - return uri, err - } - - cacheDir := filepath.Join(baseCacheDir, "openshift-install", "libvirt") - httpCacheDir := filepath.Join(cacheDir, "http") - // We used to use httpCacheDir, warn if it still exists since the user may - // want to delete it - _, err = os.Stat(httpCacheDir) - if err == nil || !os.IsNotExist(err) { - logrus.Infof("The installer no longer uses %q, it can be deleted", httpCacheDir) - } - - resp, err := http.Get(uri) - if err != nil { - return uri, err - } - if resp.StatusCode != 200 { - return uri, fmt.Errorf("%s while getting %s", resp.Status, uri) - } - defer resp.Body.Close() - - key, err := cacheKey(resp.Header.Get("ETag")) - if err != nil { - return uri, fmt.Errorf("invalid ETag for %s: %v", uri, err) - } - - imageCacheDir := filepath.Join(cacheDir, "image") - err = os.MkdirAll(imageCacheDir, 0777) - if err != nil { - return uri, err - } - - imagePath := filepath.Join(imageCacheDir, key) - _, err = os.Stat(imagePath) - if err == nil { - logrus.Debugf("Using cached OS image %q", imagePath) - } else { - if !os.IsNotExist(err) { - return uri, err - } - - err = cacheImage(resp.Body, imagePath) - if err != nil { - return uri, err - } - } - - return fmt.Sprintf("file://%s", filepath.ToSlash(imagePath)), nil -} - -func cacheKey(etag string) (key string, err error) { - if etag == "" { - return "", fmt.Errorf("caching is not supported when ETag is unset") - } - etagSections := strings.SplitN(etag, "\"", 3) - if len(etagSections) != 3 { - return "", fmt.Errorf("broken quoting: %s", etag) - } - if etagSections[0] == "W/" { - return "", fmt.Errorf("caching is not supported for weak ETags: %s", etag) - } - opaque := etagSections[1] - if opaque == "" { - return "", fmt.Errorf("caching is not supported when the opaque tag is unset: %s", etag) - } - hashed := md5.Sum([]byte(opaque)) - return hex.EncodeToString(hashed[:]), nil -} - -func cacheImage(reader io.Reader, imagePath string) (err error) { - logrus.Debugf("Unpacking OS image into %q...", imagePath) - - flockPath := fmt.Sprintf("%s.lock", imagePath) - flock, err := os.Create(flockPath) - if err != nil { - return err - } - defer flock.Close() - defer func() { - err2 := os.Remove(flockPath) - if err == nil { - err = err2 - } - }() - - err = unix.Flock(int(flock.Fd()), unix.LOCK_EX) - if err != nil { - return err - } - defer func() { - err2 := unix.Flock(int(flock.Fd()), unix.LOCK_UN) - if err == nil { - err = err2 - } - }() - - _, err = os.Stat(imagePath) - if err != nil && !os.IsNotExist(err) { - return nil // another cacheImage beat us to it - } - - tempPath := fmt.Sprintf("%s.tmp", imagePath) - - // Delete the temporary file that may have been left over from previous launches. - err = os.Remove(tempPath) - if err != nil { - if !os.IsNotExist(err) { - return fmt.Errorf("failed to clean up %s: %v", tempPath, err) - } - } else { - logrus.Debugf("Temporary file %v that remained after the previous launches was deleted", tempPath) - } - - file, err := os.OpenFile(tempPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0444) - if err != nil { - return err - } - closed := false - defer func() { - if !closed { - file.Close() - } - }() - - _, err = io.Copy(file, reader) - if err != nil { - return err - } - - err = file.Close() - if err != nil { - return err - } - closed = true - - return os.Rename(tempPath, imagePath) -} diff --git a/pkg/tfvars/libvirt/libvirt.go b/pkg/tfvars/libvirt/libvirt.go index a51fbfba11d..e4e5e4927c8 100644 --- a/pkg/tfvars/libvirt/libvirt.go +++ b/pkg/tfvars/libvirt/libvirt.go @@ -9,6 +9,7 @@ import ( "github.com/apparentlymart/go-cidr/cidr" "github.com/openshift/cluster-api-provider-libvirt/pkg/apis/libvirtproviderconfig/v1beta1" + "github.com/openshift/installer/pkg/tfvars/internal/cache" "github.com/pkg/errors" ) @@ -34,7 +35,7 @@ func TFVars(masterConfig *v1beta1.LibvirtMachineProviderConfig, osImage string, return nil, err } - osImage, err = cachedImage(osImage) + osImage, err = cache.DownloadImageFile(osImage) if err != nil { return nil, errors.Wrap(err, "failed to use cached libvirt image") } diff --git a/pkg/tfvars/openstack/openstack.go b/pkg/tfvars/openstack/openstack.go index 9d428769b1c..734070f4675 100644 --- a/pkg/tfvars/openstack/openstack.go +++ b/pkg/tfvars/openstack/openstack.go @@ -2,13 +2,8 @@ package openstack import ( - "compress/gzip" "encoding/json" "fmt" - "io" - "net/url" - "os" - "strings" "github.com/gophercloud/gophercloud/openstack/identity/v3/tokens" "github.com/gophercloud/gophercloud/openstack/imageservice/v2/images" @@ -16,7 +11,6 @@ import ( "github.com/openshift/installer/pkg/rhcos" "github.com/openshift/installer/pkg/tfvars/internal/cache" "github.com/pkg/errors" - "github.com/sirupsen/logrus" "sigs.k8s.io/cluster-api-provider-openstack/pkg/apis/openstackproviderconfig/v1alpha1" ) @@ -68,33 +62,7 @@ func TFVars(masterConfig *v1alpha1.OpenstackProviderSpec, cloud string, external if err != nil { return nil, err } - - // Compressed image support was added to OpenStack Glance only in Train release. Unfortunately previous - // versions do not have this feature, so we have to check whether or not we need to decompress the file. - // For more information: https://docs.openstack.org/glance/latest/user/formats.html - // TODO(mfedosin): Allow to skip this step if Glance supports compressed images. - baseImageURL, err := url.ParseRequestURI(baseImage) - // If the file has ".gz" extension, then its data is compressed - if strings.HasSuffix(baseImageURL.Path, ".gz") { - localFilePathUncompressed := localFilePath + ".uncompressed" - - // Do nothing if we already have the uncompressed file in cache, otherwise decompress the data - _, err = os.Stat(localFilePathUncompressed) - if err != nil { - if os.IsNotExist(err) { - logrus.Infof("Decompress image data from %v to %v", localFilePath, localFilePathUncompressed) - err = decompressFile(localFilePath, localFilePathUncompressed) - if err != nil { - return nil, err - } - } else { - return nil, err - } - } - cfg.BaseImageLocalFilePath = localFilePathUncompressed - } else { - cfg.BaseImageLocalFilePath = localFilePath - } + cfg.BaseImageLocalFilePath = localFilePath } else { // Not a URL -> use baseImage value as an overridden Glance image name. // Need to check if this image exists and there are no other images with this name. @@ -130,32 +98,6 @@ func TFVars(masterConfig *v1alpha1.OpenstackProviderSpec, cloud string, external return json.MarshalIndent(cfg, "", " ") } -// decompressFile decompresses data in the cache -func decompressFile(src, dest string) error { - gzipfile, err := os.Open(src) - if err != nil { - return err - } - - reader, err := gzip.NewReader(gzipfile) - defer reader.Close() - if err != nil { - return err - } - - writer, err := os.Create(dest) - defer writer.Close() - if err != nil { - return err - } - - if _, err = io.Copy(writer, reader); err != nil { - return err - } - - return nil -} - func validateOverriddenImageName(imageName, cloud string) error { opts := &clientconfig.ClientOpts{ Cloud: cloud,