Merge pull request #315 from bitgully/main

Allow for overriding default dependency source

dependency_cache.go

@@ -18,6 +18,7 @@ package libpak
 
 import (
 	"crypto/sha256"
+	"crypto/tls"
 	"encoding/hex"
 	"fmt"
 	"io"
@@ -46,8 +47,8 @@ type HttpClientTimeouts struct {
 	ExpectContinueTimeout time.Duration
 }
 
-// DependencyCache allows a user to get an artifact either from a buildpack's cache, a previous download, or to download
-// directly.
+// DependencyCache allows a user to get an artifact either from a buildpack's cache, a previous download,
+// a mirror registry, or to download directly.
 type DependencyCache struct {
 
 	// CachePath is the location where the buildpack has cached its dependencies.
@@ -67,19 +68,31 @@ type DependencyCache struct {
 
 	// httpClientTimeouts contains the timeout values used by HTTP client
 	HttpClientTimeouts HttpClientTimeouts
+
+	// Alternative source used for downloading dependencies.
+	DependencyMirror string
 }
 
 // NewDependencyCache creates a new instance setting the default cache path (<BUILDPACK_PATH>/dependencies) and user
 // agent (<BUILDPACK_ID>/<BUILDPACK_VERSION>).
-// Mappings will be read from any libcnb.Binding in the context with type "dependency-mappings"
+// Mappings will be read from any libcnb.Binding in the context with type "dependency-mappings".
+//
+// In some air-gapped environments, dependencies might not be download directly but need to be pulled from a local mirror registry.
+// In such cases, an alternative URI can either be provided as environment variable "BP_DEPENDENCY_MIRROR", or by a binding of type "dependency-mirror"
+// where a file named "uri" holds the desired location.
+// The two schemes https:// and file:// are supported in mirror URIs where the expected format is (optional parts in "[]"):
+// <scheme>://[<username>:<password>@]<hostname>[:<port>][/<prefix>]
+// The optional path part of the provided URI is used as a prefix that might be necessary in some setups.
+// This (prefix) path may also include a placeholder of "{originalHost}" at any level (in sub-paths or at top-level) and is replaced with the
+// hostname of the original download URI at build time. A sample mirror URI might look like this: https://local-mirror.example.com/buildpacks-dependencies/{originalHost}
 func NewDependencyCache(context libcnb.BuildContext) (DependencyCache, error) {
 	cache := DependencyCache{
 		CachePath:    filepath.Join(context.Buildpack.Path, "dependencies"),
 		DownloadPath: os.TempDir(),
 		UserAgent:    fmt.Sprintf("%s/%s", context.Buildpack.Info.ID, context.Buildpack.Info.Version),
 		Mappings:     map[string]string{},
 	}
-	mappings, err := mappingsFromBindings(context.Platform.Bindings)
+	mappings, err := filterBindingsByType(context.Platform.Bindings, "dependency-mapping")
 	if err != nil {
 		return DependencyCache{}, fmt.Errorf("unable to process dependency-mapping bindings\n%w", err)
 	}
@@ -91,6 +104,12 @@ func NewDependencyCache(context libcnb.BuildContext) (DependencyCache, error) {
 	}
 	cache.HttpClientTimeouts = *clientTimeouts
 
+	dependencyMirror, err := getDependencyMirror(context.Platform.Bindings)
+	if err != nil {
+		return DependencyCache{}, err
+	}
+	cache.DependencyMirror = dependencyMirror
+
 	return cache, nil
 }
 
@@ -134,19 +153,39 @@ func customizeHttpClientTimeouts() (*HttpClientTimeouts, error) {
 	}, nil
 }
 
-func mappingsFromBindings(bindings libcnb.Bindings) (map[string]string, error) {
-	mappings := map[string]string{}
+// Returns the URI of a dependency mirror (optional).
+// Such mirror location can be defined in a binding of type 'dependency-mirror' with filename 'uri'
+// or using the environment variable 'BP_DEPENDENCY_MIRROR'. The latter takes precedence in case both are found.
+func getDependencyMirror(bindings libcnb.Bindings) (string, error) {
+	dependencyMirror := sherpa.GetEnvWithDefault("BP_DEPENDENCY_MIRROR", "")
+	// If no mirror was found in environment variables, try to find one in bindings.
+	if dependencyMirror == "" {
+		dependencyMirrorBindings, err := filterBindingsByType(bindings, "dependency-mirror")
+		if err == nil {
+			// Use the content of the file named "uri" as the mirror's URI.
+			dependencyMirror = dependencyMirrorBindings["uri"]
+		} else {
+			return "", err
+		}
+	}
+	return dependencyMirror, nil
+}
+
+// Returns a key/value map with all entries for a given binding type.
+// An error is returned if multiple entries are found using the same key (e.g. duplicate digests in dependency mappings).
+func filterBindingsByType(bindings libcnb.Bindings, bindingType string) (map[string]string, error) {
+	filteredBindings := map[string]string{}
 	for _, binding := range bindings {
-		if strings.ToLower(binding.Type) == "dependency-mapping" {
-			for digest, uri := range binding.Secret {
-				if _, ok := mappings[digest]; ok {
-					return nil, fmt.Errorf("multiple mappings for digest %q", digest)
+		if strings.ToLower(binding.Type) == bindingType {
+			for key, value := range binding.Secret {
+				if _, ok := filteredBindings[key]; ok {
+					return nil, fmt.Errorf("multiple %s bindings found with duplicate keys %s", binding.Type, key)
 				}
-				mappings[digest] = uri
+				filteredBindings[key] = value
 			}
 		}
 	}
-	return mappings, nil
+	return filteredBindings, nil
 }
 
 // RequestModifierFunc is a callback that enables modification of a download request before it is sent.  It is often
@@ -164,15 +203,17 @@ type RequestModifierFunc func(request *http.Request) (*http.Request, error)
 func (d *DependencyCache) Artifact(dependency BuildpackDependency, mods ...RequestModifierFunc) (*os.File, error) {
 
 	var (
-		actual   BuildpackDependency
-		artifact string
-		file     string
-		uri      = dependency.URI
-		urlP     *url.URL
+		actual    BuildpackDependency
+		artifact  string
+		file      string
+		isBinding bool
+		uri       = dependency.URI
+		urlP      *url.URL
 	)
 
 	for d, u := range d.Mappings {
 		if d == dependency.SHA256 {
+			isBinding = true
 			uri = u
 			break
 		}
@@ -184,6 +225,13 @@ func (d *DependencyCache) Artifact(dependency BuildpackDependency, mods ...Reque
 		return nil, fmt.Errorf("unable to parse URI. see DEBUG log level")
 	}
 
+	if isBinding && d.DependencyMirror != "" {
+		d.Logger.Bodyf("Both dependency mirror and bindings are present. %s Please remove dependency map bindings if you wish to use the mirror.",
+			color.YellowString("Mirror is being ignored."))
+	} else {
+		d.setDependencyMirror(urlP)
+	}
+
 	if dependency.SHA256 == "" {
 		d.Logger.Headerf("%s Dependency has no SHA256. Skipping cache.",
 			color.New(color.FgYellow, color.Bold).Sprint("Warning:"))
@@ -287,6 +335,28 @@ func (d DependencyCache) downloadFile(source string, destination string, mods ..
 }
 
 func (d DependencyCache) downloadHttp(url *url.URL, destination string, mods ...RequestModifierFunc) error {
+	var httpClient *http.Client
+	if (strings.EqualFold(url.Hostname(), "localhost")) || (strings.EqualFold(url.Hostname(), "127.0.0.1")) {
+		httpClient = &http.Client{
+			Transport: &http.Transport{
+				TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+			},
+		}
+	} else {
+		httpClient = &http.Client{
+			Transport: &http.Transport{
+				Dial: (&net.Dialer{
+					Timeout:   d.HttpClientTimeouts.DialerTimeout,
+					KeepAlive: d.HttpClientTimeouts.DialerKeepAlive,
+				}).Dial,
+				TLSHandshakeTimeout:   d.HttpClientTimeouts.TLSHandshakeTimeout,
+				ResponseHeaderTimeout: d.HttpClientTimeouts.ResponseHeaderTimeout,
+				ExpectContinueTimeout: d.HttpClientTimeouts.ExpectContinueTimeout,
+				Proxy:                 http.ProxyFromEnvironment,
+			},
+		}
+	}
+
 	req, err := http.NewRequest("GET", url.String(), nil)
 	if err != nil {
 		return fmt.Errorf("unable to create new GET request for %s\n%w", url.Redacted(), err)
@@ -303,19 +373,7 @@ func (d DependencyCache) downloadHttp(url *url.URL, destination string, mods ...
 		}
 	}
 
-	client := http.Client{
-		Transport: &http.Transport{
-			Dial: (&net.Dialer{
-				Timeout:   d.HttpClientTimeouts.DialerTimeout,
-				KeepAlive: d.HttpClientTimeouts.DialerKeepAlive,
-			}).Dial,
-			TLSHandshakeTimeout:   d.HttpClientTimeouts.TLSHandshakeTimeout,
-			ResponseHeaderTimeout: d.HttpClientTimeouts.ResponseHeaderTimeout,
-			ExpectContinueTimeout: d.HttpClientTimeouts.ExpectContinueTimeout,
-			Proxy:                 http.ProxyFromEnvironment,
-		},
-	}
-	resp, err := client.Do(req)
+	resp, err := httpClient.Do(req)
 	if err != nil {
 		return fmt.Errorf("unable to request %s\n%w", url.Redacted(), err)
 	}
@@ -363,3 +421,20 @@ func (DependencyCache) verify(path string, expected string) error {
 
 	return nil
 }
+
+func (d DependencyCache) setDependencyMirror(urlD *url.URL) {
+	if d.DependencyMirror != "" {
+		d.Logger.Bodyf("%s Download URIs will be overridden.", color.GreenString("Dependency mirror found."))
+		urlOverride, err := url.ParseRequestURI(d.DependencyMirror)
+
+		if strings.ToLower(urlOverride.Scheme) == "https" || strings.ToLower(urlOverride.Scheme) == "file" {
+			urlD.Scheme = urlOverride.Scheme
+			urlD.User = urlOverride.User
+			urlD.Path = strings.Replace(urlOverride.Path, "{originalHost}", urlD.Hostname(), 1) + urlD.Path
+			urlD.Host = urlOverride.Host
+		} else {
+			d.Logger.Debugf("Dependency mirror URI is invalid: %s\n%w", d.DependencyMirror, err)
+			d.Logger.Bodyf("%s is ignored. Have you used one of the supported schemes https:// or file://?", color.YellowString("Invalid dependency mirror"))
+		}
+	}
+}

dependency_cache_test.go

@@ -150,6 +150,35 @@ func testDependencyCache(t *testing.T, context spec.G, it spec.S) {
 				})
 			})
 		})
+
+		context("dependency mirror from environment variable", func() {
+			it.Before(func() {
+				t.Setenv("BP_DEPENDENCY_MIRROR", "https://env-var-mirror.acme.com")
+			})
+
+			it("uses BP_DEPENDENCY_MIRROR environment variable", func() {
+				dependencyCache, err := libpak.NewDependencyCache(ctx)
+				Expect(err).NotTo(HaveOccurred())
+				Expect(dependencyCache.DependencyMirror).To(Equal("https://env-var-mirror.acme.com"))
+			})
+		})
+
+		context("dependency mirror from binding", func() {
+			it.Before(func() {
+				ctx.Platform.Bindings = append(ctx.Platform.Bindings, libcnb.Binding{
+					Type: "dependency-mirror",
+					Secret: map[string]string{
+						"uri": "https://bindings-mirror.acme.com",
+					},
+				})
+			})
+
+			it("uses dependency-mirror binding", func() {
+				dependencyCache, err := libpak.NewDependencyCache(ctx)
+				Expect(err).NotTo(HaveOccurred())
+				Expect(dependencyCache.DependencyMirror).To(Equal("https://bindings-mirror.acme.com"))
+			})
+		})
 	})
 
 	context("artifacts", func() {
@@ -298,6 +327,92 @@ func testDependencyCache(t *testing.T, context spec.G, it spec.S) {
 			})
 		})
 
+		context("dependency mirror is used https", func() {
+			var mirrorServer *ghttp.Server
+
+			it.Before(func() {
+				mirrorServer = ghttp.NewTLSServer()
+			})
+
+			it.After(func() {
+				mirrorServer.Close()
+			})
+
+			it("downloads from https mirror", func() {
+				url, err := url.Parse(mirrorServer.URL())
+				Expect(err).NotTo(HaveOccurred())
+				mirrorServer.AppendHandlers(ghttp.CombineHandlers(
+					ghttp.VerifyBasicAuth("username", "password"),
+					ghttp.VerifyRequest(http.MethodGet, "/foo/bar/test-path", ""),
+					ghttp.RespondWith(http.StatusOK, "test-fixture"),
+				))
+
+				dependencyCache.DependencyMirror = url.Scheme + "://" + "username:password@" + url.Host + "/foo/bar"
+				a, err := dependencyCache.Artifact(dependency)
+				Expect(err).NotTo(HaveOccurred())
+
+				Expect(io.ReadAll(a)).To(Equal([]byte("test-fixture")))
+			})
+
+			it("downloads from https mirror preserving hostname", func() {
+				url, err := url.Parse(mirrorServer.URL())
+				Expect(err).NotTo(HaveOccurred())
+				mirrorServer.AppendHandlers(ghttp.CombineHandlers(
+					ghttp.VerifyRequest(http.MethodGet, "/"+url.Hostname()+"/test-path", ""),
+					ghttp.RespondWith(http.StatusOK, "test-fixture"),
+				))
+
+				dependencyCache.DependencyMirror = url.Scheme + "://" + url.Host + "/{originalHost}"
+				a, err := dependencyCache.Artifact(dependency)
+				Expect(err).NotTo(HaveOccurred())
+
+				Expect(io.ReadAll(a)).To(Equal([]byte("test-fixture")))
+			})
+		})
+
+		context("dependency mirror is used file", func() {
+			var (
+				mirrorPath              string
+				mirrorPathPreservedHost string
+			)
+
+			it.Before(func() {
+				var err error
+				mirrorPath, err = os.MkdirTemp("", "mirror-path")
+				Expect(err).NotTo(HaveOccurred())
+				originalUrl, err := url.Parse(dependency.URI)
+				Expect(err).NotTo(HaveOccurred())
+				mirrorPathPreservedHost = filepath.Join(mirrorPath, originalUrl.Hostname(), "prefix")
+				Expect(os.MkdirAll(mirrorPathPreservedHost, os.ModePerm)).NotTo(HaveOccurred())
+			})
+
+			it.After(func() {
+				Expect(os.RemoveAll(mirrorPath)).To(Succeed())
+			})
+
+			it("downloads from file mirror", func() {
+				mirrorFile := filepath.Join(mirrorPath, "test-path")
+				Expect(os.WriteFile(mirrorFile, []byte("test-fixture"), 0644)).ToNot(HaveOccurred())
+
+				dependencyCache.DependencyMirror = "file://" + mirrorPath
+				a, err := dependencyCache.Artifact(dependency)
+				Expect(err).NotTo(HaveOccurred())
+
+				Expect(io.ReadAll(a)).To(Equal([]byte("test-fixture")))
+			})
+
+			it("downloads from file mirror preserving hostname", func() {
+				mirrorFilePreservedHost := filepath.Join(mirrorPathPreservedHost, "test-path")
+				Expect(os.WriteFile(mirrorFilePreservedHost, []byte("test-fixture"), 0644)).ToNot(HaveOccurred())
+
+				dependencyCache.DependencyMirror = "file://" + mirrorPath + "/{originalHost}" + "/prefix"
+				a, err := dependencyCache.Artifact(dependency)
+				Expect(err).NotTo(HaveOccurred())
+
+				Expect(io.ReadAll(a)).To(Equal([]byte("test-fixture")))
+			})
+		})
+
 		it("fails with invalid SHA256", func() {
 			server.AppendHandlers(ghttp.RespondWith(http.StatusOK, "invalid-fixture"))