From dfddf149399d35beb44199a9afc467ca9a4d0499 Mon Sep 17 00:00:00 2001 From: Sanskar Jaiswal Date: Mon, 29 May 2023 17:04:33 +0530 Subject: [PATCH 1/3] git/gogit: add support for per client proxying Add `WithProxy()`, a `ClientOption` which configures the proxy settings for a `gogit.Client`. All remote operations performed by the client use the configured proxy. Signed-off-by: Sanskar Jaiswal --- git/gogit/client.go | 19 ++++++++++ git/gogit/clone.go | 17 +++++---- git/gogit/clone_test.go | 80 ++++++++++++++++++++++++++++++++++++++--- 3 files changed, 105 insertions(+), 11 deletions(-) diff --git a/git/gogit/client.go b/git/gogit/client.go index a7fad648..40f7a534 100644 --- a/git/gogit/client.go +++ b/git/gogit/client.go @@ -32,6 +32,7 @@ import ( "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/plumbing/cache" "github.com/go-git/go-git/v5/plumbing/object" + "github.com/go-git/go-git/v5/plumbing/transport" "github.com/go-git/go-git/v5/storage" "github.com/go-git/go-git/v5/storage/filesystem" "github.com/go-git/go-git/v5/storage/memory" @@ -55,6 +56,7 @@ type Client struct { credentialsOverHTTP bool useDefaultKnownHosts bool singleBranch bool + proxy transport.ProxyOptions } var _ repository.Client = &Client{} @@ -95,6 +97,8 @@ func NewClient(path string, authOpts *git.AuthOptions, clientOpts ...ClientOptio return g, nil } +// WithStorer configures the client to use the provided Storer for +// storing all Git related objects. func WithStorer(s storage.Storer) ClientOption { return func(c *Client) error { c.storer = s @@ -102,6 +106,8 @@ func WithStorer(s storage.Storer) ClientOption { } } +// WithWorkTreeFS configures the client to use the provided filesystem +// for storing the worktree. func WithWorkTreeFS(wt billy.Filesystem) ClientOption { return func(c *Client) error { c.worktreeFS = wt @@ -128,6 +134,8 @@ func WithSingleBranch(singleBranch bool) ClientOption { } } +// WithDiskStorage configures the client to store the worktree and all +// Git related objects on disk. func WithDiskStorage() ClientOption { return func(c *Client) error { wt := fs.New(c.path) @@ -139,6 +147,8 @@ func WithDiskStorage() ClientOption { } } +// WithMemoryStorage configures the client to store the worktree and +// all Git related objects in memory. func WithMemoryStorage() ClientOption { return func(c *Client) error { c.storer = memory.NewStorage() @@ -165,6 +175,15 @@ func WithFallbackToDefaultKnownHosts() ClientOption { } } +// WithProxy configures the proxy settings to be used for all +// remote operations. +func WithProxy(opts transport.ProxyOptions) ClientOption { + return func(c *Client) error { + c.proxy = opts + return nil + } +} + func (g *Client) Init(ctx context.Context, url, branch string) error { if err := g.validateUrl(url); err != nil { return err diff --git a/git/gogit/clone.go b/git/gogit/clone.go index c58308c0..f856b1b8 100644 --- a/git/gogit/clone.go +++ b/git/gogit/clone.go @@ -52,7 +52,7 @@ func (g *Client) cloneBranch(ctx context.Context, url, branch string, opts repos ref := plumbing.NewBranchReferenceName(branch) // check if previous revision has changed before attempting to clone if lastObserved := git.TransformRevision(opts.LastObservedCommit); lastObserved != "" { - head, err := getRemoteHEAD(ctx, url, ref, g.authOpts, authMethod) + head, err := g.getRemoteHEAD(ctx, url, ref, authMethod) if err != nil { return nil, err } @@ -84,6 +84,7 @@ func (g *Client) cloneBranch(ctx context.Context, url, branch string, opts repos Progress: nil, Tags: extgogit.NoTags, CABundle: caBundle(g.authOpts), + ProxyOptions: g.proxy, } repo, err := extgogit.CloneContext(ctx, g.storer, g.worktreeFS, cloneOpts) @@ -134,7 +135,7 @@ func (g *Client) cloneTag(ctx context.Context, url, tag string, opts repository. ref := plumbing.NewTagReferenceName(tag) // check if previous revision has changed before attempting to clone if lastObserved := git.TransformRevision(opts.LastObservedCommit); lastObserved != "" { - head, err := getRemoteHEAD(ctx, url, ref, g.authOpts, authMethod) + head, err := g.getRemoteHEAD(ctx, url, ref, authMethod) if err != nil { return nil, err } @@ -166,6 +167,7 @@ func (g *Client) cloneTag(ctx context.Context, url, tag string, opts repository. Progress: nil, Tags: extgogit.NoTags, CABundle: caBundle(g.authOpts), + ProxyOptions: g.proxy, } repo, err := extgogit.CloneContext(ctx, g.storer, g.worktreeFS, cloneOpts) @@ -206,6 +208,7 @@ func (g *Client) cloneCommit(ctx context.Context, url, commit string, opts repos Progress: nil, Tags: extgogit.NoTags, CABundle: caBundle(g.authOpts), + ProxyOptions: g.proxy, } if opts.Branch != "" { cloneOpts.SingleBranch = g.singleBranch @@ -270,6 +273,7 @@ func (g *Client) cloneSemVer(ctx context.Context, url, semverTag string, opts re Progress: nil, Tags: extgogit.AllTags, CABundle: caBundle(g.authOpts), + ProxyOptions: g.proxy, } repo, err := extgogit.CloneContext(ctx, g.storer, g.worktreeFS, cloneOpts) @@ -373,7 +377,7 @@ func (g *Client) cloneRefName(ctx context.Context, url string, refName string, c if err != nil { return nil, fmt.Errorf("unable to construct auth method with options: %w", err) } - head, err := getRemoteHEAD(ctx, url, plumbing.ReferenceName(refName), g.authOpts, authMethod) + head, err := g.getRemoteHEAD(ctx, url, plumbing.ReferenceName(refName), authMethod) if err != nil { return nil, err } @@ -404,8 +408,8 @@ func recurseSubmodules(recurse bool) extgogit.SubmoduleRescursivity { return extgogit.NoRecurseSubmodules } -func getRemoteHEAD(ctx context.Context, url string, ref plumbing.ReferenceName, - authOpts *git.AuthOptions, authMethod transport.AuthMethod) (string, error) { +func (g *Client) getRemoteHEAD(ctx context.Context, url string, ref plumbing.ReferenceName, + authMethod transport.AuthMethod) (string, error) { // ref: https://git-scm.com/docs/git-check-ref-format#_description; point no. 6 if strings.HasPrefix(ref.String(), "/") || strings.HasSuffix(ref.String(), "/") { return "", fmt.Errorf("ref %s is invalid; Git refs cannot begin or end with a slash '/'", ref.String()) @@ -418,8 +422,9 @@ func getRemoteHEAD(ctx context.Context, url string, ref plumbing.ReferenceName, remote := extgogit.NewRemote(memory.NewStorage(), remoteCfg) listOpts := &extgogit.ListOptions{ Auth: authMethod, - CABundle: authOpts.CAFile, + CABundle: caBundle(g.authOpts), PeelingOption: extgogit.AppendPeeled, + ProxyOptions: g.proxy, } refs, err := remote.ListContext(ctx, listOpts) if err != nil { diff --git a/git/gogit/clone_test.go b/git/gogit/clone_test.go index ffc1f3ba..31b95ee9 100644 --- a/git/gogit/clone_test.go +++ b/git/gogit/clone_test.go @@ -21,6 +21,7 @@ import ( "errors" "fmt" iofs "io/fs" + "net" "net/http" "net/http/httptest" "net/url" @@ -31,12 +32,14 @@ import ( "testing" "time" + "github.com/elazarl/goproxy" "github.com/go-git/go-billy/v5/memfs" extgogit "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/config" "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/plumbing/cache" "github.com/go-git/go-git/v5/plumbing/object" + "github.com/go-git/go-git/v5/plumbing/transport" "github.com/go-git/go-git/v5/storage/filesystem" . "github.com/onsi/gomega" cryptossh "golang.org/x/crypto/ssh" @@ -1125,6 +1128,58 @@ func Test_ssh_HostKeyAlgos(t *testing.T) { } } +func TestClone_WithProxy(t *testing.T) { + g := NewWithT(t) + + server, err := gittestserver.NewTempGitServer() + g.Expect(err).ToNot(HaveOccurred()) + defer os.RemoveAll(server.Root()) + + err = server.StartHTTP() + g.Expect(err).ToNot(HaveOccurred()) + defer server.StopHTTP() + + repoPath := "proxy.git" + err = server.InitRepo("../testdata/git/repo", git.DefaultBranch, repoPath) + g.Expect(err).ToNot(HaveOccurred()) + repoURL := server.HTTPAddress() + "/" + repoPath + + proxy := goproxy.NewProxyHttpServer() + proxy.Verbose = true + var proxiedRequests int32 + setupHTTPProxy(proxy, &proxiedRequests) + + httpListener, err := net.Listen("tcp", ":0") + g.Expect(err).ToNot(HaveOccurred()) + defer httpListener.Close() + + httpProxyAddr := fmt.Sprintf("http://localhost:%d", httpListener.Addr().(*net.TCPAddr).Port) + proxyServer := http.Server{ + Addr: httpProxyAddr, + Handler: proxy, + } + go proxyServer.Serve(httpListener) + defer proxyServer.Close() + + proxyOpts := transport.ProxyOptions{ + URL: httpProxyAddr, + } + authOpts := &git.AuthOptions{ + Transport: git.HTTP, + } + ggc, err := NewClient(t.TempDir(), authOpts, WithDiskStorage(), WithProxy(proxyOpts)) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(ggc.proxy.URL).ToNot(BeEmpty()) + + _, err = ggc.Clone(context.TODO(), repoURL, repository.CloneConfig{ + CheckoutStrategy: repository.CheckoutStrategy{ + Branch: git.DefaultBranch, + }, + }) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(proxiedRequests).To(BeNumerically(">", 0)) +} + func Test_getRemoteHEAD(t *testing.T) { g := NewWithT(t) repo, path, err := initRepo(t.TempDir()) @@ -1134,7 +1189,10 @@ func Test_getRemoteHEAD(t *testing.T) { cc, err := commitFile(repo, "test", "testing current head branch", time.Now()) g.Expect(err).ToNot(HaveOccurred()) ref := plumbing.NewBranchReferenceName(git.DefaultBranch) - head, err := getRemoteHEAD(context.TODO(), path, ref, &git.AuthOptions{}, nil) + ggc, err := NewClient("", nil) + g.Expect(err).ToNot(HaveOccurred()) + + head, err := ggc.getRemoteHEAD(context.TODO(), path, ref, nil) g.Expect(err).ToNot(HaveOccurred()) g.Expect(head).To(Equal(fmt.Sprintf("refs/heads/%s@%s", git.DefaultBranch, git.Hash(cc.String()).Digest()))) @@ -1144,22 +1202,22 @@ func Test_getRemoteHEAD(t *testing.T) { g.Expect(err).ToNot(HaveOccurred()) ref = plumbing.NewTagReferenceName("v0.1.0") - head, err = getRemoteHEAD(context.TODO(), path, ref, &git.AuthOptions{}, nil) + head, err = ggc.getRemoteHEAD(context.TODO(), path, ref, nil) g.Expect(err).ToNot(HaveOccurred()) g.Expect(head).To(Equal(fmt.Sprintf("refs/tags/%s@%s", "v0.1.0", git.Hash(cc.String()).Digest()))) ref = plumbing.NewTagReferenceName("v0.1.0" + tagDereferenceSuffix) - head, err = getRemoteHEAD(context.TODO(), path, ref, &git.AuthOptions{}, nil) + head, err = ggc.getRemoteHEAD(context.TODO(), path, ref, nil) g.Expect(err).ToNot(HaveOccurred()) g.Expect(head).To(Equal(fmt.Sprintf("refs/tags/%s@%s", "v0.1.0"+tagDereferenceSuffix, git.Hash(cc.String()).Digest()))) ref = plumbing.ReferenceName("/refs/heads/main") - head, err = getRemoteHEAD(context.TODO(), path, ref, &git.AuthOptions{}, nil) + head, err = ggc.getRemoteHEAD(context.TODO(), path, ref, nil) g.Expect(err).To(HaveOccurred()) g.Expect(err.Error()).To(Equal(fmt.Sprintf("ref %s is invalid; Git refs cannot begin or end with a slash '/'", ref.String()))) ref = plumbing.ReferenceName("refs/heads/main/") - head, err = getRemoteHEAD(context.TODO(), path, ref, &git.AuthOptions{}, nil) + head, err = ggc.getRemoteHEAD(context.TODO(), path, ref, nil) g.Expect(err).To(HaveOccurred()) g.Expect(err.Error()).To(Equal(fmt.Sprintf("ref %s is invalid; Git refs cannot begin or end with a slash '/'", ref.String()))) } @@ -1518,3 +1576,15 @@ func mockSignature(time time.Time) *object.Signature { When: time, } } + +func setupHTTPProxy(proxy *goproxy.ProxyHttpServer, proxiedRequests *int32) { + var proxyHandler goproxy.FuncReqHandler = func(req *http.Request, ctx *goproxy.ProxyCtx) (*http.Request, *http.Response) { + if strings.Contains(req.Host, "127.0.0.1") { + *proxiedRequests++ + return req, nil + } + // Reject if it isn't our request. + return req, goproxy.NewResponse(req, goproxy.ContentTypeText, http.StatusForbidden, "") + } + proxy.OnRequest().Do(proxyHandler) +} From 62f49c922030a3087575c0bde16a859e929607fe Mon Sep 17 00:00:00 2001 From: Sanskar Jaiswal Date: Tue, 30 May 2023 13:33:12 +0530 Subject: [PATCH 2/3] git/gogit: update gittestserver to v0.8.4 Signed-off-by: Sanskar Jaiswal --- git/gogit/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/gogit/go.mod b/git/gogit/go.mod index 1bb6ef74..15e4ea9e 100644 --- a/git/gogit/go.mod +++ b/git/gogit/go.mod @@ -16,7 +16,7 @@ require ( github.com/elazarl/goproxy v0.0.0-20221015165544-a0805db90819 github.com/fluxcd/gitkit v0.6.0 github.com/fluxcd/pkg/git v0.12.2 - github.com/fluxcd/pkg/gittestserver v0.8.3 + github.com/fluxcd/pkg/gittestserver v0.8.4 github.com/fluxcd/pkg/ssh v0.7.4 github.com/fluxcd/pkg/version v0.2.2 github.com/go-git/go-billy/v5 v5.4.1 From 12339fbae0aefa0b8a28c11765214e38b6811b07 Mon Sep 17 00:00:00 2001 From: Sanskar Jaiswal Date: Tue, 30 May 2023 14:01:23 +0530 Subject: [PATCH 3/3] git/e2e: update deps Signed-off-by: Sanskar Jaiswal --- git/internal/e2e/go.mod | 8 ++++---- git/internal/e2e/go.sum | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/git/internal/e2e/go.mod b/git/internal/e2e/go.mod index cf19f24d..8dad72b8 100644 --- a/git/internal/e2e/go.mod +++ b/git/internal/e2e/go.mod @@ -12,10 +12,10 @@ replace ( ) require ( - github.com/fluxcd/go-git-providers v0.15.3 + github.com/fluxcd/go-git-providers v0.16.0 github.com/fluxcd/pkg/git v0.12.2 - github.com/fluxcd/pkg/git/gogit v0.9.0 - github.com/fluxcd/pkg/gittestserver v0.8.3 + github.com/fluxcd/pkg/git/gogit v0.11.1 + github.com/fluxcd/pkg/gittestserver v0.8.4 github.com/fluxcd/pkg/ssh v0.7.4 github.com/go-git/go-git/v5 v5.7.0 github.com/go-logr/logr v1.2.4 @@ -40,7 +40,7 @@ require ( github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/google/go-cmp v0.5.9 // indirect - github.com/google/go-github/v49 v49.1.0 // indirect + github.com/google/go-github/v52 v52.0.0 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 // indirect github.com/hashicorp/errwrap v1.0.0 // indirect diff --git a/git/internal/e2e/go.sum b/git/internal/e2e/go.sum index 9d12cfd7..a174add2 100644 --- a/git/internal/e2e/go.sum +++ b/git/internal/e2e/go.sum @@ -25,8 +25,8 @@ github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= github.com/fluxcd/gitkit v0.6.0 h1:iNg5LTx6ePo+Pl0ZwqHTAkhbUHxGVSY3YCxCdw7VIFg= github.com/fluxcd/gitkit v0.6.0/go.mod h1:svOHuKi0fO9HoawdK4HfHAJJseZDHHjk7I3ihnCIqNo= -github.com/fluxcd/go-git-providers v0.15.3 h1:vJ1J+WxZYxrOrWp2ojpixjERxmN6XY9C/AxQbuVaIsQ= -github.com/fluxcd/go-git-providers v0.15.3/go.mod h1:6fkRPzq0EQHQKO0/6CmfoEr6YHYwBKzDbxiEUjaxzl4= +github.com/fluxcd/go-git-providers v0.16.0 h1:egDN1uv0jyHyvtFNHE1FQ1Slj5Xu7QEFxWj1shqYYGk= +github.com/fluxcd/go-git-providers v0.16.0/go.mod h1:dIUEy97GuCKAYHkDQS39Jqb6Pfg1mnGnqA5y2mp3wR4= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/gliderlabs/ssh v0.3.5 h1:OcaySEmAQJgyYcArR+gGGTHCyE7nvhEMTlYY+Dp8CpY= @@ -53,8 +53,8 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-github/v49 v49.1.0 h1:LFkMgawGQ8dfzWLH/rNE0b3u1D3n6/dw7ZmrN3b+YFY= -github.com/google/go-github/v49 v49.1.0/go.mod h1:MUUzHPrhGniB6vUKa27y37likpipzG+BXXJbG04J334= +github.com/google/go-github/v52 v52.0.0 h1:uyGWOY+jMQ8GVGSX8dkSwCzlehU3WfdxQ7GweO/JP7M= +github.com/google/go-github/v52 v52.0.0/go.mod h1:WJV6VEEUPuMo5pXqqa2ZCZEdbQqua4zAk2MZTIo+m+4= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 h1:yAJXTCF9TqKcTiHJAE8dj7HMvPfh66eeA2JYW7eFpSE=