Git Repository Overhaul (#155)

Update the pull function so that tag-provided mirrors do not fetch all
tags, instead later fetching (externally) only the tag they need.

Implement tag fetching function to retrieve only the desired tag when
creating a tag-provided repo mirror.

Checkout the tag into a detatched HEAD state at the end of the create
stage of tag-provided repository mirrors.

Implement ref removal and addition functions used during package
creation and mirror push to ensure that refs that aren't wanted on the
mirror won't be pushed.

Update the refspecs used in the push to push detatched HEAD, branches,
online remote, and tags to the offline mirror. Note that if we later
checkout a branch from the remote and do not clean up the remote ref it
will lead to a duplicate ref name and the push will fail on one of the
refs (likely the online one since it is later in the refspec slice).

Fixes #154

feat: Allow for repos to be provided without a tag to mirror all branches/tags
feat: Make tag-provided repository mirrors use the tag as master
fix: Prevent tag-provided repo mirrors from storing extra refs
fix: tag-provided clones use trunk branch name
docs: Update gitops-data Example README

cli/internal/git/checkout.go

@@ -3,22 +3,95 @@ package git
 import (
 	"github.com/go-git/go-git/v5"
 	"github.com/go-git/go-git/v5/plumbing"
+	"github.com/go-git/go-git/v5/plumbing/object"
 	"github.com/sirupsen/logrus"
 )
 
+// CheckoutTag performs a `git checkout` of the provided tag to a detached HEAD
 func CheckoutTag(path string, tag string) {
+	options := &git.CheckoutOptions{
+		Branch: plumbing.ReferenceName("refs/tags/" + tag),
+	}
+	checkout(path, options)
+}
+
+// CheckoutTagAsBranch performs a `git checkout` of the provided tag but rather
+// than checking out to a detatched head, checks out to the provided branch ref
+// It will delete the branch provided if it exists
+func CheckoutTagAsBranch(path string, tag string, branch plumbing.ReferenceName) {
+	logContext := logrus.WithFields(logrus.Fields{
+		"Path":   path,
+		"Tag":    tag,
+		"Branch": branch.String(),
+	})
+
+	repo, err := git.PlainOpen(path)
+	if err != nil {
+		logContext.Debug(err)
+		logContext.Fatal("Not a valid git repo or unable to open")
+	}
 
+	tagRef, err := repo.Tag(tag)
+	if err != nil {
+		logContext.Debug(err)
+		logContext.Fatal("Failed to locate tag in repository.")
+	}
+	checkoutHashAsBranch(path, tagRef.Hash(), branch)
+}
+
+// checkoutHashAsBranch performs a `git checkout` of the commit hash associated
+// with the provided hash
+// It will delete the branch provided if it exists
+func checkoutHashAsBranch(path string, hash plumbing.Hash, branch plumbing.ReferenceName) {
+	logContext := logrus.WithFields(logrus.Fields{
+		"Path":   path,
+		"Hash":   hash.String(),
+		"Branch": branch.String(),
+	})
+
+	DeleteBranchIfExists(path, branch)
+
+	repo, err := git.PlainOpen(path)
+	if err != nil {
+		logContext.Debug(err)
+		logContext.Fatal("Not a valid git repo or unable to open")
+	}
+
+	objRef, err := repo.Object(plumbing.AnyObject, hash)
+
+	var commitHash plumbing.Hash
+	switch objRef := objRef.(type) {
+	case *object.Tag:
+		commitHash = objRef.Target
+	case *object.Commit:
+		commitHash = objRef.Hash
+	default:
+		// This shouldn't ever hit, but we should at least log it if someday it
+		// does get hit
+		logContext.Debug("Unsupported tag hash type: " + objRef.Type().String())
+		logContext.Fatal("Checkout failed. Hash type not supported.")
+	}
+
+	options := &git.CheckoutOptions{
+		Hash:   commitHash,
+		Branch: branch,
+		Create: true,
+	}
+	checkout(path, options)
+}
+
+// checkout performs a `git checkout` on the path provided using the options provided
+// It assumes the caller knows what to do and does not perform any safety checks
+func checkout(path string, checkoutOptions *git.CheckoutOptions) {
 	logContext := logrus.WithFields(logrus.Fields{
 		"Path": path,
-		"Tag":  tag,
 	})
 
 	// Open the given repo
 	repo, err := git.PlainOpen(path)
 	if err != nil {
 		logContext.Debug(err)
 		logContext.Fatal("Not a valid git repo or unable to open")
-		return
 	}
 
 	// Get the working tree so we can change refs
@@ -28,12 +101,10 @@ func CheckoutTag(path string, tag string) {
 		logContext.Fatal("Unable to load the git repo")
 	}
 
-	// Checkout our tag
-	err = tree.Checkout(&git.CheckoutOptions{
-		Branch: plumbing.ReferenceName("refs/tags/" + tag),
-	})
+	// Perform the checkout
+	err = tree.Checkout(checkoutOptions)
 	if err != nil {
 		logContext.Debug(err)
-		logContext.Fatal("Unable to checkout the given tag")
+		logContext.Fatal("Unable to perform checkout")
 	}
 }

cli/internal/git/fetch.go

@@ -0,0 +1,65 @@
+package git
+
+import (
+	"path"
+
+	"github.com/go-git/go-git/v5"
+	goConfig "github.com/go-git/go-git/v5/config"
+	"github.com/sirupsen/logrus"
+)
+
+// FetchTag performs a `git fetch` of _only_ the provided tag
+func FetchTag(gitDirectory string, tag string) {
+	logContext := logrus.WithFields(logrus.Fields{
+		// Base should be similar to the repo name
+		"Repo": path.Base(gitDirectory),
+	})
+
+	repo, err := git.PlainOpen(gitDirectory)
+	if err != nil {
+		logContext.Fatal(err)
+	}
+
+	remotes, err := repo.Remotes()
+	// There should never be no remotes, but it's easier to account for than
+	// let be a bug later
+	if err != nil || len(remotes) == 0 {
+		if err != nil {
+			logContext.Debug(err)
+		}
+		logContext.Fatal("Failed to identify remotes.")
+	}
+
+	gitUrl := remotes[0].Config().URLs[0]
+	// Now that we have an exact match, we may as well update the logger,
+	// especially since nothing has been logged to this point that hasn't been
+	// fatal.
+	logContext = logrus.WithFields(logrus.Fields{
+		"Remote": gitUrl,
+	})
+
+	gitCred := FindAuthForHost(gitUrl)
+
+	logContext.Debug("Attempting to find tag: " + tag)
+	fetchOptions := &git.FetchOptions{
+		RemoteName: onlineRemoteName,
+		RefSpecs: []goConfig.RefSpec{
+			goConfig.RefSpec("refs/tags/" + tag + ":refs/tags/" + tag),
+		},
+	}
+
+	if gitCred.Auth.Username != "" {
+		fetchOptions.Auth = &gitCred.Auth
+	}
+
+	err = repo.Fetch(fetchOptions)
+
+	if err == git.ErrTagExists {
+		logContext.Info("Tag already fetched")
+	} else if err != nil {
+		logContext.Debug(err)
+		logContext.Fatal("Not a valid tag or unable to fetch")
+	}
+
+	logContext.Info("Git tag fetched")
+}

cli/internal/git/pull.go

@@ -5,20 +5,27 @@ import (
 
 	"github.com/defenseunicorns/zarf/cli/internal/utils"
 	"github.com/go-git/go-git/v5"
+	"github.com/go-git/go-git/v5/plumbing"
 	"github.com/sirupsen/logrus"
+
+	"strings"
 )
 
 const onlineRemoteName = "online-upstream"
 
 func DownloadRepoToTemp(gitUrl string) string {
 	path := utils.MakeTempDir()
+	// If downloading to temp, grab all tags since the repo isn't being
+	// packaged anyways and it saves us from having to fetch the tags
+	// later if we need them
 	pull(gitUrl, path)
 	return path
 }
 
-func Pull(gitUrl string, targetFolder string) {
+func Pull(gitUrl string, targetFolder string) string {
 	path := targetFolder + "/" + transformURLtoRepoName(gitUrl)
 	pull(gitUrl, path)
+	return path
 }
 
 func pull(gitUrl string, targetFolder string) {
@@ -29,19 +36,25 @@ func pull(gitUrl string, targetFolder string) {
 
 	gitCred := FindAuthForHost(gitUrl)
 
+	matches := strings.Split(gitUrl, "@")
+	fetchAllTags := len(matches) == 1
 	cloneOptions := &git.CloneOptions{
-		URL:        gitUrl,
+		URL:        matches[0],
 		Progress:   os.Stdout,
 		RemoteName: onlineRemoteName,
 	}
 
+	if !fetchAllTags {
+		cloneOptions.Tags = git.NoTags
+	}
+
 	// Gracefully handle no git creds on the system (like our CI/CD)
 	if gitCred.Auth.Username != "" {
 		cloneOptions.Auth = &gitCred.Auth
 	}
 
 	// Clone the given repo
-	_, err := git.PlainClone(targetFolder, false, cloneOptions)
+	repo, err := git.PlainClone(targetFolder, false, cloneOptions)
 
 	if err == git.ErrRepositoryAlreadyExists {
 		logContext.Info("Repo already cloned")
@@ -50,5 +63,31 @@ func pull(gitUrl string, targetFolder string) {
 		logContext.Fatal("Not a valid git repo or unable to clone")
 	}
 
+	if !fetchAllTags {
+		tag := matches[1]
+
+		// Identify the remote trunk branch name
+		trunkBranchName := plumbing.NewBranchReferenceName("master")
+		head, err := repo.Head()
+
+		if err != nil {
+			// No repo head available
+			logContext.Debug(err)
+			logContext.Warn("Failed to identify repo head. Tag will be pushed to 'master'.")
+		} else if head.Name().IsBranch() {
+			// Valid repo head and it is a branch
+			trunkBranchName = head.Name()
+		} else {
+			// Valid repo head but not a branch
+			logContext.Warn("No branch found for this repo head. Tag will be pushed to 'master'.")
+		}
+
+		RemoveLocalBranchRefs(targetFolder)
+		RemoveOnlineRemoteRefs(targetFolder)
+
+		FetchTag(targetFolder, tag)
+		CheckoutTagAsBranch(targetFolder, tag, trunkBranchName)
+	}
+
 	logContext.Info("Git repo synced")
 }

cli/internal/git/push.go

@@ -1,6 +1,8 @@
 package git
 
 import (
+	"os"
+
 	"github.com/defenseunicorns/zarf/cli/config"
 	"github.com/defenseunicorns/zarf/cli/internal/utils"
 	"github.com/go-git/go-git/v5"
@@ -9,6 +11,7 @@ import (
 )
 
 const offlineRemoteName = "offline-downstream"
+const onlineRemoteRefPrefix = "refs/remotes/" + onlineRemoteName + "/"
 
 func PushAllDirectories(localPath string) {
 	paths := utils.ListDirectories(localPath)
@@ -25,8 +28,7 @@ func push(localPath string) {
 	// Open the given repo
 	repo, err := git.PlainOpen(localPath)
 	if err != nil {
-		logContext.Warn("Not a valid git repo or unable to open")
-		return
+		logContext.Fatal("Not a valid git repo or unable to open")
 	}
 
 	// Get the upstream URL
@@ -38,29 +40,47 @@ func push(localPath string) {
 	remoteUrl := remote.Config().URLs[0]
 	targetUrl := transformURL("https://"+config.ZarfLocalIP, remoteUrl)
 
-	_, _ = repo.CreateRemote(&goConfig.RemoteConfig{
+	_, err = repo.CreateRemote(&goConfig.RemoteConfig{
 		Name: offlineRemoteName,
 		URLs: []string{targetUrl},
 	})
 
+	if err != nil {
+		logContext.Debug(err)
+		logContext.Fatal("Failed to create offline remote")
+	}
+
 	gitCred := FindAuthForHost(config.ZarfLocalIP)
 
+	pushContext := logContext.WithField("target", targetUrl)
+
+	// Since we are pushing HEAD:refs/heads/master on deployment, leaving
+	// duplicates of the HEAD ref (ex. refs/heads/master,
+	// refs/remotes/online-upstream/master, will cause the push to fail)
+	removedRefs := RemoveHeadCopies(localPath)
+
 	err = repo.Push(&git.PushOptions{
 		RemoteName: offlineRemoteName,
 		Auth:       &gitCred.Auth,
+		Progress:   os.Stdout,
+		// If a provided refspec doesn't push anything, it is just ignored
 		RefSpecs: []goConfig.RefSpec{
 			"refs/heads/*:refs/heads/*",
+			onlineRemoteRefPrefix + "*:refs/heads/*",
 			"refs/tags/*:refs/tags/*",
 		},
 	})
 
-	pushContext := logContext.WithField("target", targetUrl)
 	if err == git.NoErrAlreadyUpToDate {
 		pushContext.Info("Repo already up-to-date")
 	} else if err != nil {
+		pushContext.Debug(err)
 		pushContext.Warn("Unable to push repo to the gitops service")
 	} else {
 		pushContext.Info("Repo updated")
 	}
 
+	// Add back the refs we removed just incase this push isn't the last thing
+	// being run and a later task needs to reference them.
+	AddRefs(localPath, removedRefs)
 }