Merge pull request #280 from wking/cache-libvirt-image

installer/pkg/config/libvirt: Caching for libvirt pulls

installer/pkg/workflow/BUILD.bazel

@@ -16,7 +16,6 @@ go_library(
     visibility = ["//visibility:public"],
     deps = [
         "//installer/pkg/config-generator:go_default_library",
-        "//installer/pkg/copy:go_default_library",
         "//pkg/types/config:go_default_library",
         "//vendor/github.com/Sirupsen/logrus:go_default_library",
         "//vendor/gopkg.in/yaml.v2:go_default_library",

installer/pkg/workflow/init.go

@@ -10,7 +10,6 @@ import (
 	yaml "gopkg.in/yaml.v2"
 
 	configgenerator "github.com/openshift/installer/installer/pkg/config-generator"
-	"github.com/openshift/installer/installer/pkg/copy"
 	"github.com/openshift/installer/pkg/types/config"
 )
 
@@ -93,6 +92,12 @@ func prepareWorspaceStep(m *metadata) error {
 		return err
 	}
 
+	if cluster.Platform == config.PlatformLibvirt {
+		if err := cluster.Libvirt.UseCachedImage(); err != nil {
+			return err
+		}
+	}
+
 	// generate clusterDir folder
 	clusterDir := filepath.Join(dir, cluster.Name)
 	m.clusterDir = clusterDir
@@ -106,7 +111,11 @@ func prepareWorspaceStep(m *metadata) error {
 
 	// put config file under the clusterDir folder
 	configFilePath := filepath.Join(clusterDir, configFileName)
-	if err := copy.Copy(m.configFilePath, configFilePath); err != nil {
+	configContent, err := yaml.Marshal(cluster)
+	if err != nil {
+		return err
+	}
+	if err := ioutil.WriteFile(configFilePath, configContent, 0666); err != nil {
 		return fmt.Errorf("failed to create cluster config at %q: %v", clusterDir, err)
 	}
 

pkg/types/config/libvirt/BUILD.bazel

@@ -2,8 +2,15 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library")
 
 go_library(
     name = "go_default_library",
-    srcs = ["libvirt.go"],
+    srcs = [
+        "cache.go",
+        "libvirt.go",
+    ],
     importpath = "github.com/openshift/installer/pkg/types/config/libvirt",
     visibility = ["//visibility:public"],
-    deps = ["//vendor/github.com/apparentlymart/go-cidr/cidr:go_default_library"],
+    deps = [
+        "//vendor/github.com/apparentlymart/go-cidr/cidr:go_default_library",
+        "//vendor/github.com/gregjones/httpcache:go_default_library",
+        "//vendor/github.com/gregjones/httpcache/diskcache:go_default_library",
+    ],
 )

pkg/types/config/libvirt/cache.go

@@ -0,0 +1,81 @@
+package libvirt
+
+import (
+	"compress/gzip"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/gregjones/httpcache"
+	"github.com/gregjones/httpcache/diskcache"
+)
+
+// UseCachedImage leaves non-file:// image URIs unalterered.
+// Other URIs are retrieved with a local cache at
+// $XDG_CACHE_HOME/openshift-install/libvirt [1].  This allows you to
+// use the same remote image URI multiple times without needing to
+// worry about redundant downloads, although you will want to
+// periodically blow away your cache.
+//
+// [1]: https://standards.freedesktop.org/basedir-spec/basedir-spec-0.7.html
+func (libvirt *Libvirt) UseCachedImage() (err error) {
+	// FIXME: set the default URI here?  Leave it elsewhere?
+
+	if strings.HasPrefix(libvirt.Image, "file://") {
+		return nil
+	}
+
+	// FIXME: Use os.UserCacheDir() once we bump to Go 1.11
+	// baseCacheDir, err := os.UserCacheDir()
+	// if err != nil {
+	// 	return err
+	// }
+	baseCacheDir := filepath.Join(os.Getenv("HOME"), ".cache")
+
+	cacheDir := filepath.Join(baseCacheDir, "openshift-install", "libvirt")
+	err = os.MkdirAll(cacheDir, 0777)
+	if err != nil {
+		return err
+	}
+
+	cache := diskcache.New(cacheDir)
+	transport := httpcache.NewTransport(cache)
+	resp, err := transport.Client().Get(libvirt.Image)
+	if err != nil {
+		return err
+	}
+	if resp.StatusCode != 200 {
+		return fmt.Errorf("%s while getting %s", resp.Status, libvirt.Image)
+	}
+	defer resp.Body.Close()
+
+	var reader io.Reader
+	if strings.HasSuffix(libvirt.Image, ".gz") {
+		reader, err = gzip.NewReader(resp.Body)
+		if err != nil {
+			return err
+		}
+	} else {
+		reader = resp.Body
+	}
+
+	// FIXME: diskcache's diskv backend doesn't expose direct file access.
+	// We can write our own cache implementation to get around this,
+	// but for now just dump this into /tmp without cleanup.
+	file, err := ioutil.TempFile("", "openshift-install-")
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	_, err = io.Copy(file, reader)
+	if err != nil {
+		return err
+	}
+
+	libvirt.Image = fmt.Sprintf("file://%s", filepath.ToSlash(file.Name()))
+	return nil
+}

pkg/types/config/parser.go

@@ -31,6 +31,7 @@ func ParseConfig(data []byte) (*Cluster, error) {
 			return nil, err
 		}
 		cluster.PullSecret = string(data)
+		cluster.PullSecretPath = ""
 	}
 
 	if cluster.Platform == PlatformAWS && cluster.EC2AMIOverride == "" {