From 503f6318162f2fb83b9063aa69073e3000ccd7cb Mon Sep 17 00:00:00 2001 From: bufdev <4228796+bufdev@users.noreply.github.com> Date: Fri, 13 Aug 2021 13:41:20 -0400 Subject: [PATCH] Use bare clones for git cloner (#421) --- go.mod | 4 +-- go.sum | 8 +++--- internal/pkg/git/cloner.go | 49 ++++++++++++++++-------------------- internal/pkg/git/git_test.go | 14 +++++++++++ 4 files changed, 42 insertions(+), 33 deletions(-) diff --git a/go.mod b/go.mod index 2134b883bb..ef8f72f3c3 100644 --- a/go.mod +++ b/go.mod @@ -20,11 +20,11 @@ require ( go.opencensus.io v0.23.0 go.uber.org/multierr v1.7.0 go.uber.org/zap v1.19.0 - golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d // indirect + golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d // indirect golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e // indirect golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 golang.org/x/text v0.3.7 // indirect - google.golang.org/genproto v0.0.0-20210811021853-ddbe55d93216 // indirect + google.golang.org/genproto v0.0.0-20210813162853-db860fec028c // indirect google.golang.org/grpc v1.40.0-dev.0.20210708170655-30dfb4b933a5 google.golang.org/protobuf v1.27.1 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect diff --git a/go.sum b/go.sum index 3310e8d3bc..7edd5c9448 100644 --- a/go.sum +++ b/go.sum @@ -361,8 +361,8 @@ golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d h1:20cMwl2fHAzkJMEA+8J4JgqBQcQGzbisXo31MIeenXI= -golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d h1:LO7XpTYMwTqxjLcGWPijK3vRXg1aWdlNOVOHRq45d7c= +golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -576,8 +576,8 @@ google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210811021853-ddbe55d93216 h1:qnrhhl4uoNFepTqE28u11llFcDH07Z6r/cQxpGR97A4= -google.golang.org/genproto v0.0.0-20210811021853-ddbe55d93216/go.mod h1:cFeNkxwySK631ADgubI+/XFU/xp8FD5KIVV4rj8UC5w= +google.golang.org/genproto v0.0.0-20210813162853-db860fec028c h1:iLQakcwWG3k/++1q/46apVb1sUQ3IqIdn9yUE6eh/xA= +google.golang.org/genproto v0.0.0-20210813162853-db860fec028c/go.mod h1:cFeNkxwySK631ADgubI+/XFU/xp8FD5KIVV4rj8UC5w= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= diff --git a/internal/pkg/git/cloner.go b/internal/pkg/git/cloner.go index a862ecbad0..fc64652d23 100644 --- a/internal/pkg/git/cloner.go +++ b/internal/pkg/git/cloner.go @@ -32,11 +32,6 @@ import ( "go.uber.org/zap" ) -// bufCloneOrigin is the name for the remote. It helps distinguish the origin of -// the repo we're cloning from the "origin" of our clone (which is the repo -// being cloned). -const bufCloneOrigin = "bufCloneOrigin" - type cloner struct { logger *zap.Logger storageosProvider storageos.Provider @@ -83,30 +78,28 @@ func (c *cloner) CloneToBucket( depthArg := strconv.Itoa(int(depth)) - tmpDir, err := tmp.NewDir() + bareDir, err := tmp.NewDir() if err != nil { return err } defer func() { - retErr = multierr.Append(retErr, tmpDir.Close()) + retErr = multierr.Append(retErr, bareDir.Close()) }() - - buffer := bytes.NewBuffer(nil) - cmd := exec.CommandContext(ctx, "git", "init") - cmd.Dir = tmpDir.AbsPath() - cmd.Env = app.Environ(envContainer) - cmd.Stderr = buffer - if err := cmd.Run(); err != nil { - return newGitCommandError(err, buffer, tmpDir) + worktreeDir, err := tmp.NewDir() + if err != nil { + return err } + defer func() { + retErr = multierr.Append(retErr, worktreeDir.Close()) + }() - buffer.Reset() - cmd = exec.CommandContext(ctx, "git", "remote", "add", bufCloneOrigin, url) - cmd.Dir = tmpDir.AbsPath() + buffer := bytes.NewBuffer(nil) + cmd := exec.CommandContext(ctx, "git", "init", "--bare") + cmd.Dir = bareDir.AbsPath() cmd.Env = app.Environ(envContainer) cmd.Stderr = buffer if err := cmd.Run(); err != nil { - return newGitCommandError(err, buffer, tmpDir) + return newGitCommandError(err, buffer, bareDir) } var gitConfigAuthArgs []string @@ -122,9 +115,10 @@ func (c *cloner) CloneToBucket( fetchRef, checkoutRefs := getRefspecsForName(options.Name) fetchArgs := append( gitConfigAuthArgs, + "--git-dir="+bareDir.AbsPath(), "fetch", "--depth", depthArg, - bufCloneOrigin, + url, fetchRef, ) @@ -137,32 +131,34 @@ func (c *cloner) CloneToBucket( buffer.Reset() cmd = exec.CommandContext(ctx, "git", fetchArgs...) - cmd.Dir = tmpDir.AbsPath() cmd.Env = app.Environ(envContainer) cmd.Stderr = buffer if err := cmd.Run(); err != nil { - return newGitCommandError(err, buffer, tmpDir) + return newGitCommandError(err, buffer, bareDir) } for _, checkoutRef := range checkoutRefs { buffer.Reset() args := append( gitConfigAuthArgs, + "--git-dir="+bareDir.AbsPath(), + "--work-tree="+worktreeDir.AbsPath(), "checkout", checkoutRef, ) cmd = exec.CommandContext(ctx, "git", args...) - cmd.Dir = tmpDir.AbsPath() cmd.Env = app.Environ(envContainer) cmd.Stderr = buffer if err := cmd.Run(); err != nil { - return newGitCommandError(err, buffer, tmpDir) + return newGitCommandError(err, buffer, worktreeDir) } } if options.RecurseSubmodules { submoduleArgs := append( gitConfigAuthArgs, + "--git-dir="+bareDir.AbsPath(), + "--work-tree="+worktreeDir.AbsPath(), "submodule", "update", "--init", @@ -173,16 +169,15 @@ func (c *cloner) CloneToBucket( buffer.Reset() cmd = exec.CommandContext(ctx, "git", submoduleArgs...) cmd.Env = app.Environ(envContainer) - cmd.Dir = tmpDir.AbsPath() cmd.Stderr = buffer if err := cmd.Run(); err != nil { // Suppress printing of temp path - return fmt.Errorf("%v\n%v", err, strings.Replace(buffer.String(), tmpDir.AbsPath(), "", -1)) + return fmt.Errorf("%v\n%v", err, strings.Replace(buffer.String(), worktreeDir.AbsPath(), "", -1)) } } // we do NOT want to read in symlinks - tmpReadWriteBucket, err := c.storageosProvider.NewReadWriteBucket(tmpDir.AbsPath()) + tmpReadWriteBucket, err := c.storageosProvider.NewReadWriteBucket(worktreeDir.AbsPath()) if err != nil { return err } diff --git a/internal/pkg/git/git_test.go b/internal/pkg/git/git_test.go index 2c321ba85c..a59c4aeeb5 100644 --- a/internal/pkg/git/git_test.go +++ b/internal/pkg/git/git_test.go @@ -91,6 +91,17 @@ func TestGitCloner(t *testing.T) { assert.True(t, storage.IsNotExist(err)) }) + t.Run("branch_and_main_ref", func(t *testing.T) { + t.Parallel() + readBucket := readBucketForName(t, workDir, 2, NewRefNameWithBranch("HEAD~", "main")) + + content, err := storage.ReadPath(context.Background(), readBucket, "test.proto") + require.NoError(t, err) + assert.Equal(t, "// commit 0", string(content)) + _, err = readBucket.Stat(context.Background(), "nonexistent") + assert.True(t, storage.IsNotExist(err)) + }) + t.Run("branch_and_ref", func(t *testing.T) { t.Parallel() readBucket := readBucketForName(t, workDir, 2, NewRefNameWithBranch("local-branch~", "local-branch")) @@ -173,6 +184,9 @@ func createGitDirs(t *testing.T) (string, string) { runCommand(t, "git", "-C", originPath, "config", "user.email", "tests@buf.build") runCommand(t, "git", "-C", originPath, "config", "user.name", "Buf go tests") runCommand(t, "git", "-C", originPath, "checkout", "-b", "main") + require.NoError(t, os.WriteFile(filepath.Join(originPath, "test.proto"), []byte("// commit 0"), 0600)) + runCommand(t, "git", "-C", originPath, "add", "test.proto") + runCommand(t, "git", "-C", originPath, "commit", "-m", "commit 0") require.NoError(t, os.WriteFile(filepath.Join(originPath, "test.proto"), []byte("// commit 1"), 0600)) runCommand(t, "git", "-C", originPath, "add", "test.proto") runCommand(t, "git", "-C", originPath, "commit", "-m", "commit 1")