Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

git/gogit: Add support for per client proxying #575

Merged
merged 3 commits into from
May 30, 2023
Merged

git/gogit: Add support for per client proxying #575

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
dfddf14
git/gogit: add support for per client proxying
aryan9600 May 29, 2023
62f49c9
git/gogit: update gittestserver to v0.8.4
aryan9600 May 30, 2023
12339fb
git/e2e: update deps
aryan9600 May 30, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 19 additions & 0 deletions git/gogit/client.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ import (
"github.com/go-git/go-git/v5/plumbing"
"github.com/go-git/go-git/v5/plumbing/cache"
"github.com/go-git/go-git/v5/plumbing/object"
"github.com/go-git/go-git/v5/plumbing/transport"
"github.com/go-git/go-git/v5/storage"
"github.com/go-git/go-git/v5/storage/filesystem"
"github.com/go-git/go-git/v5/storage/memory"
Expand All @@ -55,6 +56,7 @@ type Client struct {
credentialsOverHTTP bool
useDefaultKnownHosts bool
singleBranch bool
proxy transport.ProxyOptions
}

var _ repository.Client = &Client{}
Expand Down Expand Up @@ -95,13 +97,17 @@ func NewClient(path string, authOpts *git.AuthOptions, clientOpts ...ClientOptio
return g, nil
}

// WithStorer configures the client to use the provided Storer for
// storing all Git related objects.
func WithStorer(s storage.Storer) ClientOption {
return func(c *Client) error {
c.storer = s
return nil
}
}

// WithWorkTreeFS configures the client to use the provided filesystem
// for storing the worktree.
func WithWorkTreeFS(wt billy.Filesystem) ClientOption {
return func(c *Client) error {
c.worktreeFS = wt
Expand All @@ -128,6 +134,8 @@ func WithSingleBranch(singleBranch bool) ClientOption {
}
}

// WithDiskStorage configures the client to store the worktree and all
// Git related objects on disk.
func WithDiskStorage() ClientOption {
return func(c *Client) error {
wt := fs.New(c.path)
Expand All @@ -139,6 +147,8 @@ func WithDiskStorage() ClientOption {
}
}

// WithMemoryStorage configures the client to store the worktree and
// all Git related objects in memory.
func WithMemoryStorage() ClientOption {
return func(c *Client) error {
c.storer = memory.NewStorage()
Expand All @@ -165,6 +175,15 @@ func WithFallbackToDefaultKnownHosts() ClientOption {
}
}

// WithProxy configures the proxy settings to be used for all
// remote operations.
func WithProxy(opts transport.ProxyOptions) ClientOption {
return func(c *Client) error {
c.proxy = opts
return nil
}
}

func (g *Client) Init(ctx context.Context, url, branch string) error {
if err := g.validateUrl(url); err != nil {
return err
Expand Down
17 changes: 11 additions & 6 deletions git/gogit/clone.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ func (g *Client) cloneBranch(ctx context.Context, url, branch string, opts repos
ref := plumbing.NewBranchReferenceName(branch)
// check if previous revision has changed before attempting to clone
if lastObserved := git.TransformRevision(opts.LastObservedCommit); lastObserved != "" {
head, err := getRemoteHEAD(ctx, url, ref, g.authOpts, authMethod)
head, err := g.getRemoteHEAD(ctx, url, ref, authMethod)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -84,6 +84,7 @@ func (g *Client) cloneBranch(ctx context.Context, url, branch string, opts repos
Progress: nil,
Tags: extgogit.NoTags,
CABundle: caBundle(g.authOpts),
ProxyOptions: g.proxy,
darkowlzz marked this conversation as resolved.
Show resolved Hide resolved
}

repo, err := extgogit.CloneContext(ctx, g.storer, g.worktreeFS, cloneOpts)
Expand Down Expand Up @@ -134,7 +135,7 @@ func (g *Client) cloneTag(ctx context.Context, url, tag string, opts repository.
ref := plumbing.NewTagReferenceName(tag)
// check if previous revision has changed before attempting to clone
if lastObserved := git.TransformRevision(opts.LastObservedCommit); lastObserved != "" {
head, err := getRemoteHEAD(ctx, url, ref, g.authOpts, authMethod)
head, err := g.getRemoteHEAD(ctx, url, ref, authMethod)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -166,6 +167,7 @@ func (g *Client) cloneTag(ctx context.Context, url, tag string, opts repository.
Progress: nil,
Tags: extgogit.NoTags,
CABundle: caBundle(g.authOpts),
ProxyOptions: g.proxy,
}

repo, err := extgogit.CloneContext(ctx, g.storer, g.worktreeFS, cloneOpts)
Expand Down Expand Up @@ -206,6 +208,7 @@ func (g *Client) cloneCommit(ctx context.Context, url, commit string, opts repos
Progress: nil,
Tags: extgogit.NoTags,
CABundle: caBundle(g.authOpts),
ProxyOptions: g.proxy,
}
if opts.Branch != "" {
cloneOpts.SingleBranch = g.singleBranch
Expand Down Expand Up @@ -270,6 +273,7 @@ func (g *Client) cloneSemVer(ctx context.Context, url, semverTag string, opts re
Progress: nil,
Tags: extgogit.AllTags,
CABundle: caBundle(g.authOpts),
ProxyOptions: g.proxy,
}

repo, err := extgogit.CloneContext(ctx, g.storer, g.worktreeFS, cloneOpts)
Expand Down Expand Up @@ -373,7 +377,7 @@ func (g *Client) cloneRefName(ctx context.Context, url string, refName string, c
if err != nil {
return nil, fmt.Errorf("unable to construct auth method with options: %w", err)
}
head, err := getRemoteHEAD(ctx, url, plumbing.ReferenceName(refName), g.authOpts, authMethod)
head, err := g.getRemoteHEAD(ctx, url, plumbing.ReferenceName(refName), authMethod)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -404,8 +408,8 @@ func recurseSubmodules(recurse bool) extgogit.SubmoduleRescursivity {
return extgogit.NoRecurseSubmodules
}

func getRemoteHEAD(ctx context.Context, url string, ref plumbing.ReferenceName,
authOpts *git.AuthOptions, authMethod transport.AuthMethod) (string, error) {
func (g *Client) getRemoteHEAD(ctx context.Context, url string, ref plumbing.ReferenceName,
authMethod transport.AuthMethod) (string, error) {
Comment on lines +411 to +412
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Looking at the usage of getRemoteHEAD(), it looks like the authMethod can now be computed just once per client as it needs authOpts and useDefaultKnownHosts which are both properties of the client and don't seem to change in any of the operations.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thinking more about the other options like url, all of the g.clone*() methods set the g.repository at the very end. This repository contains the remote address with a default remote name "origin". If these clone operations store the repository info on the client, other methods of the client should be able to default to the available repository. So, the url can become optional for getRemoteHEAD(). It should be able to use the default repository when the passed in URL is empty. The benefit of keeping the URL input is to allow querying other remotes if needed.

// ref: https://git-scm.com/docs/git-check-ref-format#_description; point no. 6
if strings.HasPrefix(ref.String(), "/") || strings.HasSuffix(ref.String(), "/") {
return "", fmt.Errorf("ref %s is invalid; Git refs cannot begin or end with a slash '/'", ref.String())
Expand All @@ -418,8 +422,9 @@ func getRemoteHEAD(ctx context.Context, url string, ref plumbing.ReferenceName,
remote := extgogit.NewRemote(memory.NewStorage(), remoteCfg)
listOpts := &extgogit.ListOptions{
Auth: authMethod,
CABundle: authOpts.CAFile,
CABundle: caBundle(g.authOpts),
PeelingOption: extgogit.AppendPeeled,
ProxyOptions: g.proxy,
}
refs, err := remote.ListContext(ctx, listOpts)
if err != nil {
Expand Down
80 changes: 75 additions & 5 deletions git/gogit/clone_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import (
"errors"
"fmt"
iofs "io/fs"
"net"
"net/http"
"net/http/httptest"
"net/url"
Expand All @@ -31,12 +32,14 @@ import (
"testing"
"time"

"github.com/elazarl/goproxy"
"github.com/go-git/go-billy/v5/memfs"
extgogit "github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/config"
"github.com/go-git/go-git/v5/plumbing"
"github.com/go-git/go-git/v5/plumbing/cache"
"github.com/go-git/go-git/v5/plumbing/object"
"github.com/go-git/go-git/v5/plumbing/transport"
"github.com/go-git/go-git/v5/storage/filesystem"
. "github.com/onsi/gomega"
cryptossh "golang.org/x/crypto/ssh"
Expand Down Expand Up @@ -1125,6 +1128,58 @@ func Test_ssh_HostKeyAlgos(t *testing.T) {
}
}

func TestClone_WithProxy(t *testing.T) {
g := NewWithT(t)

server, err := gittestserver.NewTempGitServer()
g.Expect(err).ToNot(HaveOccurred())
defer os.RemoveAll(server.Root())

err = server.StartHTTP()
g.Expect(err).ToNot(HaveOccurred())
defer server.StopHTTP()

repoPath := "proxy.git"
err = server.InitRepo("../testdata/git/repo", git.DefaultBranch, repoPath)
g.Expect(err).ToNot(HaveOccurred())
repoURL := server.HTTPAddress() + "/" + repoPath

proxy := goproxy.NewProxyHttpServer()
proxy.Verbose = true
var proxiedRequests int32
setupHTTPProxy(proxy, &proxiedRequests)

httpListener, err := net.Listen("tcp", ":0")
g.Expect(err).ToNot(HaveOccurred())
defer httpListener.Close()

httpProxyAddr := fmt.Sprintf("http://localhost:%d", httpListener.Addr().(*net.TCPAddr).Port)
proxyServer := http.Server{
Addr: httpProxyAddr,
Handler: proxy,
}
go proxyServer.Serve(httpListener)
defer proxyServer.Close()

proxyOpts := transport.ProxyOptions{
URL: httpProxyAddr,
}
authOpts := &git.AuthOptions{
Transport: git.HTTP,
}
ggc, err := NewClient(t.TempDir(), authOpts, WithDiskStorage(), WithProxy(proxyOpts))
g.Expect(err).ToNot(HaveOccurred())
g.Expect(ggc.proxy.URL).ToNot(BeEmpty())

_, err = ggc.Clone(context.TODO(), repoURL, repository.CloneConfig{
CheckoutStrategy: repository.CheckoutStrategy{
Branch: git.DefaultBranch,
},
})
g.Expect(err).ToNot(HaveOccurred())
g.Expect(proxiedRequests).To(BeNumerically(">", 0))
}

func Test_getRemoteHEAD(t *testing.T) {
g := NewWithT(t)
repo, path, err := initRepo(t.TempDir())
Expand All @@ -1134,7 +1189,10 @@ func Test_getRemoteHEAD(t *testing.T) {
cc, err := commitFile(repo, "test", "testing current head branch", time.Now())
g.Expect(err).ToNot(HaveOccurred())
ref := plumbing.NewBranchReferenceName(git.DefaultBranch)
head, err := getRemoteHEAD(context.TODO(), path, ref, &git.AuthOptions{}, nil)
ggc, err := NewClient("", nil)
g.Expect(err).ToNot(HaveOccurred())

head, err := ggc.getRemoteHEAD(context.TODO(), path, ref, nil)
g.Expect(err).ToNot(HaveOccurred())
g.Expect(head).To(Equal(fmt.Sprintf("refs/heads/%s@%s", git.DefaultBranch, git.Hash(cc.String()).Digest())))

Expand All @@ -1144,22 +1202,22 @@ func Test_getRemoteHEAD(t *testing.T) {
g.Expect(err).ToNot(HaveOccurred())

ref = plumbing.NewTagReferenceName("v0.1.0")
head, err = getRemoteHEAD(context.TODO(), path, ref, &git.AuthOptions{}, nil)
head, err = ggc.getRemoteHEAD(context.TODO(), path, ref, nil)
g.Expect(err).ToNot(HaveOccurred())
g.Expect(head).To(Equal(fmt.Sprintf("refs/tags/%s@%s", "v0.1.0", git.Hash(cc.String()).Digest())))

ref = plumbing.NewTagReferenceName("v0.1.0" + tagDereferenceSuffix)
head, err = getRemoteHEAD(context.TODO(), path, ref, &git.AuthOptions{}, nil)
head, err = ggc.getRemoteHEAD(context.TODO(), path, ref, nil)
g.Expect(err).ToNot(HaveOccurred())
g.Expect(head).To(Equal(fmt.Sprintf("refs/tags/%s@%s", "v0.1.0"+tagDereferenceSuffix, git.Hash(cc.String()).Digest())))

ref = plumbing.ReferenceName("/refs/heads/main")
head, err = getRemoteHEAD(context.TODO(), path, ref, &git.AuthOptions{}, nil)
head, err = ggc.getRemoteHEAD(context.TODO(), path, ref, nil)
g.Expect(err).To(HaveOccurred())
g.Expect(err.Error()).To(Equal(fmt.Sprintf("ref %s is invalid; Git refs cannot begin or end with a slash '/'", ref.String())))

ref = plumbing.ReferenceName("refs/heads/main/")
head, err = getRemoteHEAD(context.TODO(), path, ref, &git.AuthOptions{}, nil)
head, err = ggc.getRemoteHEAD(context.TODO(), path, ref, nil)
g.Expect(err).To(HaveOccurred())
g.Expect(err.Error()).To(Equal(fmt.Sprintf("ref %s is invalid; Git refs cannot begin or end with a slash '/'", ref.String())))
}
Expand Down Expand Up @@ -1518,3 +1576,15 @@ func mockSignature(time time.Time) *object.Signature {
When: time,
}
}

func setupHTTPProxy(proxy *goproxy.ProxyHttpServer, proxiedRequests *int32) {
var proxyHandler goproxy.FuncReqHandler = func(req *http.Request, ctx *goproxy.ProxyCtx) (*http.Request, *http.Response) {
if strings.Contains(req.Host, "127.0.0.1") {
*proxiedRequests++
return req, nil
}
// Reject if it isn't our request.
return req, goproxy.NewResponse(req, goproxy.ContentTypeText, http.StatusForbidden, "")
}
proxy.OnRequest().Do(proxyHandler)
}
2 changes: 1 addition & 1 deletion git/gogit/go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ require (
github.com/elazarl/goproxy v0.0.0-20221015165544-a0805db90819
github.com/fluxcd/gitkit v0.6.0
github.com/fluxcd/pkg/git v0.12.2
github.com/fluxcd/pkg/gittestserver v0.8.3
github.com/fluxcd/pkg/gittestserver v0.8.4
github.com/fluxcd/pkg/ssh v0.7.4
github.com/fluxcd/pkg/version v0.2.2
github.com/go-git/go-billy/v5 v5.4.1
Expand Down
8 changes: 4 additions & 4 deletions git/internal/e2e/go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,10 +12,10 @@ replace (
)

require (
github.com/fluxcd/go-git-providers v0.15.3
github.com/fluxcd/go-git-providers v0.16.0
github.com/fluxcd/pkg/git v0.12.2
github.com/fluxcd/pkg/git/gogit v0.9.0
github.com/fluxcd/pkg/gittestserver v0.8.3
github.com/fluxcd/pkg/git/gogit v0.11.1
github.com/fluxcd/pkg/gittestserver v0.8.4
github.com/fluxcd/pkg/ssh v0.7.4
github.com/go-git/go-git/v5 v5.7.0
github.com/go-logr/logr v1.2.4
Expand All @@ -40,7 +40,7 @@ require (
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/google/go-cmp v0.5.9 // indirect
github.com/google/go-github/v49 v49.1.0 // indirect
github.com/google/go-github/v52 v52.0.0 // indirect
github.com/google/go-querystring v1.1.0 // indirect
github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 // indirect
github.com/hashicorp/errwrap v1.0.0 // indirect
Expand Down
8 changes: 4 additions & 4 deletions git/internal/e2e/go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,8 @@ github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc
github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ=
github.com/fluxcd/gitkit v0.6.0 h1:iNg5LTx6ePo+Pl0ZwqHTAkhbUHxGVSY3YCxCdw7VIFg=
github.com/fluxcd/gitkit v0.6.0/go.mod h1:svOHuKi0fO9HoawdK4HfHAJJseZDHHjk7I3ihnCIqNo=
github.com/fluxcd/go-git-providers v0.15.3 h1:vJ1J+WxZYxrOrWp2ojpixjERxmN6XY9C/AxQbuVaIsQ=
github.com/fluxcd/go-git-providers v0.15.3/go.mod h1:6fkRPzq0EQHQKO0/6CmfoEr6YHYwBKzDbxiEUjaxzl4=
github.com/fluxcd/go-git-providers v0.16.0 h1:egDN1uv0jyHyvtFNHE1FQ1Slj5Xu7QEFxWj1shqYYGk=
github.com/fluxcd/go-git-providers v0.16.0/go.mod h1:dIUEy97GuCKAYHkDQS39Jqb6Pfg1mnGnqA5y2mp3wR4=
github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
github.com/gliderlabs/ssh v0.3.5 h1:OcaySEmAQJgyYcArR+gGGTHCyE7nvhEMTlYY+Dp8CpY=
Expand All @@ -53,8 +53,8 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-github/v49 v49.1.0 h1:LFkMgawGQ8dfzWLH/rNE0b3u1D3n6/dw7ZmrN3b+YFY=
github.com/google/go-github/v49 v49.1.0/go.mod h1:MUUzHPrhGniB6vUKa27y37likpipzG+BXXJbG04J334=
github.com/google/go-github/v52 v52.0.0 h1:uyGWOY+jMQ8GVGSX8dkSwCzlehU3WfdxQ7GweO/JP7M=
github.com/google/go-github/v52 v52.0.0/go.mod h1:WJV6VEEUPuMo5pXqqa2ZCZEdbQqua4zAk2MZTIo+m+4=
github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 h1:yAJXTCF9TqKcTiHJAE8dj7HMvPfh66eeA2JYW7eFpSE=
Expand Down