Merge pull request #366 from bitgully/main

Add skip-path Feature for Dependency Mirrors

dependency_cache.go

@@ -91,7 +91,7 @@ func NewDependencyCache(context libcnb.BuildContext) (DependencyCache, error) {
 		// We create the logger here because the initialization process may log some warnings that should be visible to users.
 		// This goes against the usual pattern, which has the user supply the Logger after initialization.
 		// There's no choice though, if we want the warning messages to be visible to users. We should clean this up in v2.
-		Logger:            bard.NewLogger(os.Stdout),
+		Logger: bard.NewLogger(os.Stdout),
 	}
 	mappings, err := filterBindingsByType(context.Platform.Bindings, "dependency-mapping")
 	if err != nil {
@@ -454,16 +454,52 @@ func (DependencyCache) verify(path string, expected string) error {
 func (d DependencyCache) setDependencyMirror(urlD *url.URL, mirror string) {
 	if mirror != "" {
 		d.Logger.Bodyf("%s Download URIs will be overridden.", color.GreenString("Dependency mirror found."))
-		urlOverride, err := url.ParseRequestURI(mirror)
+		mirrorArgs := parseMirror(mirror)
+		urlOverride, err := url.ParseRequestURI(mirrorArgs["mirror"])
 
 		if strings.ToLower(urlOverride.Scheme) == "https" || strings.ToLower(urlOverride.Scheme) == "file" {
 			urlD.Scheme = urlOverride.Scheme
 			urlD.User = urlOverride.User
-			urlD.Path = strings.Replace(urlOverride.Path, "{originalHost}", urlD.Hostname(), 1) + urlD.Path
+			urlD.Path = strings.Replace(urlOverride.Path, "{originalHost}", urlD.Hostname(), 1) + strings.Replace(urlD.Path, mirrorArgs["skip-path"], "", 1)
 			urlD.Host = urlOverride.Host
 		} else {
 			d.Logger.Debugf("Dependency mirror URI is invalid: %s\n%w", mirror, err)
 			d.Logger.Bodyf("%s is ignored. Have you used one of the supported schemes https:// or file://?", color.YellowString("Invalid dependency mirror"))
 		}
 	}
 }
+
+// Parses a raw mirror string into a map of arguments.
+func parseMirror(mirror string) map[string]string {
+
+	mirrorArgs := map[string]string{
+		"mirror":    mirror,
+		"skip-path": "",
+	}
+
+	// Split mirror string at commas and extract specified arguments.
+	for _, arg := range strings.Split(mirror, ",") {
+		argPair := strings.SplitN(arg, "=", 2)
+		// If a URI is provided without the key 'mirror=', still treat it as the 'mirror' argument.
+		// This addresses backwards compatibility and user experience as most mirrors won't need any additional arguments.
+		if len(argPair) == 1 && (strings.HasPrefix(argPair[0], "https") || strings.HasPrefix(argPair[0], "file")) {
+			mirrorArgs["mirror"] = argPair[0]
+		}
+		// Add all provided arguments to key/value map.
+		if len(argPair) == 2 {
+			mirrorArgs[argPair[0]] = argPair[1]
+		}
+	}
+
+	// Unescape mirror arguments to support URL-encoded strings.
+	tmp, err := url.PathUnescape(mirrorArgs["mirror"])
+	if err == nil {
+		mirrorArgs["mirror"] = tmp
+	}
+	tmp, err = url.PathUnescape(mirrorArgs["skip-path"])
+	if err == nil {
+		mirrorArgs["skip-path"] = tmp
+	}
+
+	return mirrorArgs
+}

dependency_cache_test.go

@@ -440,6 +440,81 @@ func testDependencyCache(t *testing.T, context spec.G, it spec.S) {
 			})
 		})
 
+		context("dependency mirror with additional arguments", func() {
+			var mirrorServer *ghttp.Server
+
+			it.Before(func() {
+				mirrorServer = ghttp.NewTLSServer()
+				dependencyCache.DependencyMirrors = map[string]string{}
+			})
+
+			it.After(func() {
+				mirrorServer.Close()
+			})
+
+			it("downloads from escaped mirror", func() {
+				mirrorUrl, err := url.Parse(mirrorServer.URL())
+				Expect(err).NotTo(HaveOccurred())
+				mirrorServer.AppendHandlers(ghttp.CombineHandlers(
+					ghttp.VerifyBasicAuth("user", "pa$$word,"),
+					ghttp.VerifyRequest(http.MethodGet, "/escaped/test-path", ""),
+					ghttp.RespondWith(http.StatusOK, "test-fixture"),
+				))
+
+				dependencyCache.DependencyMirrors["127.0.0.1"] = mirrorUrl.Scheme + "://user%3Apa%24%24word%2C%40" + mirrorUrl.Host + "%2Fescaped"
+				a, err := dependencyCache.Artifact(dependency)
+				Expect(err).NotTo(HaveOccurred())
+
+				Expect(io.ReadAll(a)).To(Equal([]byte("test-fixture")))
+			})
+
+			it("respects skip-path argument without mirror= key", func() {
+				mirrorUrl, err := url.Parse(mirrorServer.URL())
+				Expect(err).NotTo(HaveOccurred())
+				mirrorServer.AppendHandlers(ghttp.CombineHandlers(
+					ghttp.VerifyRequest(http.MethodGet, "/test-skip", ""),
+					ghttp.RespondWith(http.StatusOK, "test-fixture"),
+				))
+
+				dependencyCache.DependencyMirrors["127.0.0.1"] = mirrorUrl.Scheme + "://" + mirrorUrl.Host + "/test-skip,skip-path=/test-path"
+				a, err := dependencyCache.Artifact(dependency)
+				Expect(err).NotTo(HaveOccurred())
+
+				Expect(io.ReadAll(a)).To(Equal([]byte("test-fixture")))
+			})
+
+			it("respects skip-path argument with mirror= key", func() {
+				mirrorUrl, err := url.Parse(mirrorServer.URL())
+				Expect(err).NotTo(HaveOccurred())
+				mirrorServer.AppendHandlers(ghttp.CombineHandlers(
+					ghttp.VerifyRequest(http.MethodGet, "/test-skip", ""),
+					ghttp.RespondWith(http.StatusOK, "test-fixture"),
+				))
+
+				dependencyCache.DependencyMirrors["127.0.0.1"] = "mirror=" + mirrorUrl.Scheme + "://" + mirrorUrl.Host + "/test-skip,skip-path=/test-path"
+				a, err := dependencyCache.Artifact(dependency)
+				Expect(err).NotTo(HaveOccurred())
+
+				Expect(io.ReadAll(a)).To(Equal([]byte("test-fixture")))
+			})
+
+			it("respects skip-path argument when URL encoded", func() {
+				mirrorUrl, err := url.Parse(mirrorServer.URL())
+				Expect(err).NotTo(HaveOccurred())
+				mirrorServer.AppendHandlers(ghttp.CombineHandlers(
+					ghttp.VerifyRequest(http.MethodGet, "/test-skip", ""),
+					ghttp.RespondWith(http.StatusOK, "test-fixture"),
+				))
+
+				dependencyCache.DependencyMirrors["127.0.0.1"] = mirrorUrl.Scheme + "://" + mirrorUrl.Host + "/test-skip,skip-path=/test%2Cpath"
+				dependency.URI = fmt.Sprintf("%s/test,path", server.URL())
+				a, err := dependencyCache.Artifact(dependency)
+				Expect(err).NotTo(HaveOccurred())
+
+				Expect(io.ReadAll(a)).To(Equal([]byte("test-fixture")))
+			})
+		})
+
 		it("fails with invalid SHA256", func() {
 			server.AppendHandlers(ghttp.RespondWith(http.StatusOK, "invalid-fixture"))