Merge pull request #575 from fluxcd/gogit-proxy

git/gogit: Add support for per client proxying

git/gogit/client.go

@@ -32,6 +32,7 @@ import (
 	"github.com/go-git/go-git/v5/plumbing"
 	"github.com/go-git/go-git/v5/plumbing/cache"
 	"github.com/go-git/go-git/v5/plumbing/object"
+	"github.com/go-git/go-git/v5/plumbing/transport"
 	"github.com/go-git/go-git/v5/storage"
 	"github.com/go-git/go-git/v5/storage/filesystem"
 	"github.com/go-git/go-git/v5/storage/memory"
@@ -55,6 +56,7 @@ type Client struct {
 	credentialsOverHTTP  bool
 	useDefaultKnownHosts bool
 	singleBranch         bool
+	proxy                transport.ProxyOptions
 }
 
 var _ repository.Client = &Client{}
@@ -95,13 +97,17 @@ func NewClient(path string, authOpts *git.AuthOptions, clientOpts ...ClientOptio
 	return g, nil
 }
 
+// WithStorer configures the client to use the provided Storer for
+// storing all Git related objects.
 func WithStorer(s storage.Storer) ClientOption {
 	return func(c *Client) error {
 		c.storer = s
 		return nil
 	}
 }
 
+// WithWorkTreeFS configures the client to use the provided filesystem
+// for storing the worktree.
 func WithWorkTreeFS(wt billy.Filesystem) ClientOption {
 	return func(c *Client) error {
 		c.worktreeFS = wt
@@ -128,6 +134,8 @@ func WithSingleBranch(singleBranch bool) ClientOption {
 	}
 }
 
+// WithDiskStorage configures the client to store the worktree and all
+// Git related objects on disk.
 func WithDiskStorage() ClientOption {
 	return func(c *Client) error {
 		wt := fs.New(c.path)
@@ -139,6 +147,8 @@ func WithDiskStorage() ClientOption {
 	}
 }
 
+// WithMemoryStorage configures the client to store the worktree and
+// all Git related objects in memory.
 func WithMemoryStorage() ClientOption {
 	return func(c *Client) error {
 		c.storer = memory.NewStorage()
@@ -165,6 +175,15 @@ func WithFallbackToDefaultKnownHosts() ClientOption {
 	}
 }
 
+// WithProxy configures the proxy settings to be used for all
+// remote operations.
+func WithProxy(opts transport.ProxyOptions) ClientOption {
+	return func(c *Client) error {
+		c.proxy = opts
+		return nil
+	}
+}
+
 func (g *Client) Init(ctx context.Context, url, branch string) error {
 	if err := g.validateUrl(url); err != nil {
 		return err

git/gogit/clone.go

@@ -52,7 +52,7 @@ func (g *Client) cloneBranch(ctx context.Context, url, branch string, opts repos
 	ref := plumbing.NewBranchReferenceName(branch)
 	// check if previous revision has changed before attempting to clone
 	if lastObserved := git.TransformRevision(opts.LastObservedCommit); lastObserved != "" {
-		head, err := getRemoteHEAD(ctx, url, ref, g.authOpts, authMethod)
+		head, err := g.getRemoteHEAD(ctx, url, ref, authMethod)
 		if err != nil {
 			return nil, err
 		}
@@ -84,6 +84,7 @@ func (g *Client) cloneBranch(ctx context.Context, url, branch string, opts repos
 		Progress:          nil,
 		Tags:              extgogit.NoTags,
 		CABundle:          caBundle(g.authOpts),
+		ProxyOptions:      g.proxy,
 	}
 
 	repo, err := extgogit.CloneContext(ctx, g.storer, g.worktreeFS, cloneOpts)
@@ -134,7 +135,7 @@ func (g *Client) cloneTag(ctx context.Context, url, tag string, opts repository.
 	ref := plumbing.NewTagReferenceName(tag)
 	// check if previous revision has changed before attempting to clone
 	if lastObserved := git.TransformRevision(opts.LastObservedCommit); lastObserved != "" {
-		head, err := getRemoteHEAD(ctx, url, ref, g.authOpts, authMethod)
+		head, err := g.getRemoteHEAD(ctx, url, ref, authMethod)
 		if err != nil {
 			return nil, err
 		}
@@ -166,6 +167,7 @@ func (g *Client) cloneTag(ctx context.Context, url, tag string, opts repository.
 		Progress:          nil,
 		Tags:              extgogit.NoTags,
 		CABundle:          caBundle(g.authOpts),
+		ProxyOptions:      g.proxy,
 	}
 
 	repo, err := extgogit.CloneContext(ctx, g.storer, g.worktreeFS, cloneOpts)
@@ -206,6 +208,7 @@ func (g *Client) cloneCommit(ctx context.Context, url, commit string, opts repos
 		Progress:          nil,
 		Tags:              extgogit.NoTags,
 		CABundle:          caBundle(g.authOpts),
+		ProxyOptions:      g.proxy,
 	}
 	if opts.Branch != "" {
 		cloneOpts.SingleBranch = g.singleBranch
@@ -270,6 +273,7 @@ func (g *Client) cloneSemVer(ctx context.Context, url, semverTag string, opts re
 		Progress:          nil,
 		Tags:              extgogit.AllTags,
 		CABundle:          caBundle(g.authOpts),
+		ProxyOptions:      g.proxy,
 	}
 
 	repo, err := extgogit.CloneContext(ctx, g.storer, g.worktreeFS, cloneOpts)
@@ -373,7 +377,7 @@ func (g *Client) cloneRefName(ctx context.Context, url string, refName string, c
 	if err != nil {
 		return nil, fmt.Errorf("unable to construct auth method with options: %w", err)
 	}
-	head, err := getRemoteHEAD(ctx, url, plumbing.ReferenceName(refName), g.authOpts, authMethod)
+	head, err := g.getRemoteHEAD(ctx, url, plumbing.ReferenceName(refName), authMethod)
 	if err != nil {
 		return nil, err
 	}
@@ -404,8 +408,8 @@ func recurseSubmodules(recurse bool) extgogit.SubmoduleRescursivity {
 	return extgogit.NoRecurseSubmodules
 }
 
-func getRemoteHEAD(ctx context.Context, url string, ref plumbing.ReferenceName,
-	authOpts *git.AuthOptions, authMethod transport.AuthMethod) (string, error) {
+func (g *Client) getRemoteHEAD(ctx context.Context, url string, ref plumbing.ReferenceName,
+	authMethod transport.AuthMethod) (string, error) {
 	// ref: https://git-scm.com/docs/git-check-ref-format#_description; point no. 6
 	if strings.HasPrefix(ref.String(), "/") || strings.HasSuffix(ref.String(), "/") {
 		return "", fmt.Errorf("ref %s is invalid; Git refs cannot begin or end with a slash '/'", ref.String())
@@ -418,8 +422,9 @@ func getRemoteHEAD(ctx context.Context, url string, ref plumbing.ReferenceName,
 	remote := extgogit.NewRemote(memory.NewStorage(), remoteCfg)
 	listOpts := &extgogit.ListOptions{
 		Auth:          authMethod,
-		CABundle:      authOpts.CAFile,
+		CABundle:      caBundle(g.authOpts),
 		PeelingOption: extgogit.AppendPeeled,
+		ProxyOptions:  g.proxy,
 	}
 	refs, err := remote.ListContext(ctx, listOpts)
 	if err != nil {

git/gogit/clone_test.go

@@ -21,6 +21,7 @@ import (
 	"errors"
 	"fmt"
 	iofs "io/fs"
+	"net"
 	"net/http"
 	"net/http/httptest"
 	"net/url"
@@ -31,12 +32,14 @@ import (
 	"testing"
 	"time"
 
+	"github.com/elazarl/goproxy"
 	"github.com/go-git/go-billy/v5/memfs"
 	extgogit "github.com/go-git/go-git/v5"
 	"github.com/go-git/go-git/v5/config"
 	"github.com/go-git/go-git/v5/plumbing"
 	"github.com/go-git/go-git/v5/plumbing/cache"
 	"github.com/go-git/go-git/v5/plumbing/object"
+	"github.com/go-git/go-git/v5/plumbing/transport"
 	"github.com/go-git/go-git/v5/storage/filesystem"
 	. "github.com/onsi/gomega"
 	cryptossh "golang.org/x/crypto/ssh"
@@ -1125,6 +1128,58 @@ func Test_ssh_HostKeyAlgos(t *testing.T) {
 	}
 }
 
+func TestClone_WithProxy(t *testing.T) {
+	g := NewWithT(t)
+
+	server, err := gittestserver.NewTempGitServer()
+	g.Expect(err).ToNot(HaveOccurred())
+	defer os.RemoveAll(server.Root())
+
+	err = server.StartHTTP()
+	g.Expect(err).ToNot(HaveOccurred())
+	defer server.StopHTTP()
+
+	repoPath := "proxy.git"
+	err = server.InitRepo("../testdata/git/repo", git.DefaultBranch, repoPath)
+	g.Expect(err).ToNot(HaveOccurred())
+	repoURL := server.HTTPAddress() + "/" + repoPath
+
+	proxy := goproxy.NewProxyHttpServer()
+	proxy.Verbose = true
+	var proxiedRequests int32
+	setupHTTPProxy(proxy, &proxiedRequests)
+
+	httpListener, err := net.Listen("tcp", ":0")
+	g.Expect(err).ToNot(HaveOccurred())
+	defer httpListener.Close()
+
+	httpProxyAddr := fmt.Sprintf("http://localhost:%d", httpListener.Addr().(*net.TCPAddr).Port)
+	proxyServer := http.Server{
+		Addr:    httpProxyAddr,
+		Handler: proxy,
+	}
+	go proxyServer.Serve(httpListener)
+	defer proxyServer.Close()
+
+	proxyOpts := transport.ProxyOptions{
+		URL: httpProxyAddr,
+	}
+	authOpts := &git.AuthOptions{
+		Transport: git.HTTP,
+	}
+	ggc, err := NewClient(t.TempDir(), authOpts, WithDiskStorage(), WithProxy(proxyOpts))
+	g.Expect(err).ToNot(HaveOccurred())
+	g.Expect(ggc.proxy.URL).ToNot(BeEmpty())
+
+	_, err = ggc.Clone(context.TODO(), repoURL, repository.CloneConfig{
+		CheckoutStrategy: repository.CheckoutStrategy{
+			Branch: git.DefaultBranch,
+		},
+	})
+	g.Expect(err).ToNot(HaveOccurred())
+	g.Expect(proxiedRequests).To(BeNumerically(">", 0))
+}
+
 func Test_getRemoteHEAD(t *testing.T) {
 	g := NewWithT(t)
 	repo, path, err := initRepo(t.TempDir())
@@ -1134,7 +1189,10 @@ func Test_getRemoteHEAD(t *testing.T) {
 	cc, err := commitFile(repo, "test", "testing current head branch", time.Now())
 	g.Expect(err).ToNot(HaveOccurred())
 	ref := plumbing.NewBranchReferenceName(git.DefaultBranch)
-	head, err := getRemoteHEAD(context.TODO(), path, ref, &git.AuthOptions{}, nil)
+	ggc, err := NewClient("", nil)
+	g.Expect(err).ToNot(HaveOccurred())
+
+	head, err := ggc.getRemoteHEAD(context.TODO(), path, ref, nil)
 	g.Expect(err).ToNot(HaveOccurred())
 	g.Expect(head).To(Equal(fmt.Sprintf("refs/heads/%s@%s", git.DefaultBranch, git.Hash(cc.String()).Digest())))
 
@@ -1144,22 +1202,22 @@ func Test_getRemoteHEAD(t *testing.T) {
 	g.Expect(err).ToNot(HaveOccurred())
 
 	ref = plumbing.NewTagReferenceName("v0.1.0")
-	head, err = getRemoteHEAD(context.TODO(), path, ref, &git.AuthOptions{}, nil)
+	head, err = ggc.getRemoteHEAD(context.TODO(), path, ref, nil)
 	g.Expect(err).ToNot(HaveOccurred())
 	g.Expect(head).To(Equal(fmt.Sprintf("refs/tags/%s@%s", "v0.1.0", git.Hash(cc.String()).Digest())))
 
 	ref = plumbing.NewTagReferenceName("v0.1.0" + tagDereferenceSuffix)
-	head, err = getRemoteHEAD(context.TODO(), path, ref, &git.AuthOptions{}, nil)
+	head, err = ggc.getRemoteHEAD(context.TODO(), path, ref, nil)
 	g.Expect(err).ToNot(HaveOccurred())
 	g.Expect(head).To(Equal(fmt.Sprintf("refs/tags/%s@%s", "v0.1.0"+tagDereferenceSuffix, git.Hash(cc.String()).Digest())))
 
 	ref = plumbing.ReferenceName("/refs/heads/main")
-	head, err = getRemoteHEAD(context.TODO(), path, ref, &git.AuthOptions{}, nil)
+	head, err = ggc.getRemoteHEAD(context.TODO(), path, ref, nil)
 	g.Expect(err).To(HaveOccurred())
 	g.Expect(err.Error()).To(Equal(fmt.Sprintf("ref %s is invalid; Git refs cannot begin or end with a slash '/'", ref.String())))
 
 	ref = plumbing.ReferenceName("refs/heads/main/")
-	head, err = getRemoteHEAD(context.TODO(), path, ref, &git.AuthOptions{}, nil)
+	head, err = ggc.getRemoteHEAD(context.TODO(), path, ref, nil)
 	g.Expect(err).To(HaveOccurred())
 	g.Expect(err.Error()).To(Equal(fmt.Sprintf("ref %s is invalid; Git refs cannot begin or end with a slash '/'", ref.String())))
 }
@@ -1518,3 +1576,15 @@ func mockSignature(time time.Time) *object.Signature {
 		When:  time,
 	}
 }
+
+func setupHTTPProxy(proxy *goproxy.ProxyHttpServer, proxiedRequests *int32) {
+	var proxyHandler goproxy.FuncReqHandler = func(req *http.Request, ctx *goproxy.ProxyCtx) (*http.Request, *http.Response) {
+		if strings.Contains(req.Host, "127.0.0.1") {
+			*proxiedRequests++
+			return req, nil
+		}
+		// Reject if it isn't our request.
+		return req, goproxy.NewResponse(req, goproxy.ContentTypeText, http.StatusForbidden, "")
+	}
+	proxy.OnRequest().Do(proxyHandler)
+}

git/gogit/go.mod

@@ -16,7 +16,7 @@ require (
 	github.com/elazarl/goproxy v0.0.0-20221015165544-a0805db90819
 	github.com/fluxcd/gitkit v0.6.0
 	github.com/fluxcd/pkg/git v0.12.2
-	github.com/fluxcd/pkg/gittestserver v0.8.3
+	github.com/fluxcd/pkg/gittestserver v0.8.4
 	github.com/fluxcd/pkg/ssh v0.7.4
 	github.com/fluxcd/pkg/version v0.2.2
 	github.com/go-git/go-billy/v5 v5.4.1

git/internal/e2e/go.mod

@@ -12,10 +12,10 @@ replace (
 )
 
 require (
-	github.com/fluxcd/go-git-providers v0.15.3
+	github.com/fluxcd/go-git-providers v0.16.0
 	github.com/fluxcd/pkg/git v0.12.2
-	github.com/fluxcd/pkg/git/gogit v0.9.0
-	github.com/fluxcd/pkg/gittestserver v0.8.3
+	github.com/fluxcd/pkg/git/gogit v0.11.1
+	github.com/fluxcd/pkg/gittestserver v0.8.4
 	github.com/fluxcd/pkg/ssh v0.7.4
 	github.com/go-git/go-git/v5 v5.7.0
 	github.com/go-logr/logr v1.2.4
@@ -40,7 +40,7 @@ require (
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/google/go-cmp v0.5.9 // indirect
-	github.com/google/go-github/v49 v49.1.0 // indirect
+	github.com/google/go-github/v52 v52.0.0 // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
 	github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 // indirect
 	github.com/hashicorp/errwrap v1.0.0 // indirect

git/internal/e2e/go.sum

@@ -25,8 +25,8 @@ github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc
 github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ=
 github.com/fluxcd/gitkit v0.6.0 h1:iNg5LTx6ePo+Pl0ZwqHTAkhbUHxGVSY3YCxCdw7VIFg=
 github.com/fluxcd/gitkit v0.6.0/go.mod h1:svOHuKi0fO9HoawdK4HfHAJJseZDHHjk7I3ihnCIqNo=
-github.com/fluxcd/go-git-providers v0.15.3 h1:vJ1J+WxZYxrOrWp2ojpixjERxmN6XY9C/AxQbuVaIsQ=
-github.com/fluxcd/go-git-providers v0.15.3/go.mod h1:6fkRPzq0EQHQKO0/6CmfoEr6YHYwBKzDbxiEUjaxzl4=
+github.com/fluxcd/go-git-providers v0.16.0 h1:egDN1uv0jyHyvtFNHE1FQ1Slj5Xu7QEFxWj1shqYYGk=
+github.com/fluxcd/go-git-providers v0.16.0/go.mod h1:dIUEy97GuCKAYHkDQS39Jqb6Pfg1mnGnqA5y2mp3wR4=
 github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
 github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
 github.com/gliderlabs/ssh v0.3.5 h1:OcaySEmAQJgyYcArR+gGGTHCyE7nvhEMTlYY+Dp8CpY=
@@ -53,8 +53,8 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
 github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
-github.com/google/go-github/v49 v49.1.0 h1:LFkMgawGQ8dfzWLH/rNE0b3u1D3n6/dw7ZmrN3b+YFY=
-github.com/google/go-github/v49 v49.1.0/go.mod h1:MUUzHPrhGniB6vUKa27y37likpipzG+BXXJbG04J334=
+github.com/google/go-github/v52 v52.0.0 h1:uyGWOY+jMQ8GVGSX8dkSwCzlehU3WfdxQ7GweO/JP7M=
+github.com/google/go-github/v52 v52.0.0/go.mod h1:WJV6VEEUPuMo5pXqqa2ZCZEdbQqua4zAk2MZTIo+m+4=
 github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
 github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
 github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 h1:yAJXTCF9TqKcTiHJAE8dj7HMvPfh66eeA2JYW7eFpSE=