Support git repo caching on package create (#785)

## Description

Add the ability to locally cache large git repositories when pulling
them with Zarf

## Related Issue

Fixes #750

## Type of change

- [X] New feature (non-breaking change which adds functionality)

## Checklist before merging

- [x] Tests have been added/updated as necessary (add the `needs-tests`
label)
- [x] Documentation has been updated as necessary (add the `needs-docs`
label)
- [x] An ADR has been written as necessary (add the `needs-adr` label) [
[1](https://github.com/joelparkerhenderson/architecture-decision-record)
[2](https://cognitect.com/blog/2011/11/15/documenting-architecture-decisions)
[3](https://adr.github.io/) ]
- [x] (Optional) Changes have been linted locally with
[golangci-lint](https://github.com/golangci/golangci-lint). (NOTE: We
haven't turned on lint checks in the pipeline yet so linting may be hard
if it shows a lot of lint errors in places that weren't touched by
changes. Thus, linting is optional right now.)

Co-authored-by: Jon Perry <[email protected]>
Co-authored-by: Megamind <[email protected]>

Makefile

@@ -16,7 +16,7 @@ ifneq ($(UNAME_S),Linux)
 	endif
 endif
 
-AGENT_IMAGE ?= zarfdev/agent:a57bb136f21441c66630403412c6f03fc7f9cd49
+AGENT_IMAGE ?= zarfdev/agent:ef08e77c3880ac64c3cc10ecd314be0869f8f70e
 
 CLI_VERSION := $(if $(shell git describe --tags),$(shell git describe --tags),"UnknownVersion")
 BUILD_ARGS := -s -w -X 'github.com/defenseunicorns/zarf/src/config.CLIVersion=$(CLI_VERSION)'

docs/4-user-guide/1-the-zarf-cli/100-cli-commands/zarf_package_create.md

@@ -22,7 +22,7 @@ zarf package create [DIRECTORY] [flags]
       --set stringToString        Specify package variables to set on the command line (KEY=value) (default [])
       --skip-sbom                 Skip generating SBOM for this package
       --tmpdir string             Specify the temporary directory to use for intermediate files
-      --zarf-cache string         Specify the location of the Zarf image cache (default ".zarf-image-cache")
+      --zarf-cache string         Specify the location of the Zarf artifact cache (images and git repositories) (default "~/.zarf-cache")
 ```
 
 ### Options inherited from parent commands

docs/4-user-guide/1-the-zarf-cli/100-cli-commands/zarf_tools.md

@@ -20,6 +20,7 @@ Collection of additional tools to make airgap easier
 
 * [zarf](zarf.md)	 - DevSecOps Airgap Toolkit
 * [zarf tools archiver](zarf_tools_archiver.md)	 - Compress/Decompress tools for Zarf packages
+* [zarf tools clear-cache](zarf_tools_clear-cache.md)	 - Clears the configured git and image cache directory
 * [zarf tools get-git-password](zarf_tools_get-git-password.md)	 - Returns the push user's password for the Git server
 * [zarf tools monitor](zarf_tools_monitor.md)	 - Launch K9s tool for managing K8s clusters
 * [zarf tools registry](zarf_tools_registry.md)	 - Collection of registry commands provided by Crane

docs/4-user-guide/1-the-zarf-cli/100-cli-commands/zarf_tools_clear-cache.md

@@ -0,0 +1,27 @@
+## zarf tools clear-cache
+
+Clears the configured git and image cache directory
+
+```
+zarf tools clear-cache [flags]
+```
+
+### Options
+
+```
+  -h, --help                help for clear-cache
+      --zarf-cache string   Specify the location of the Zarf  artifact cache (images and git repositories) (default "~/.zarf-cache")
+```
+
+### Options inherited from parent commands
+
+```
+  -a, --architecture string   Architecture for OCI images
+  -l, --log-level string      Log level when running Zarf. Valid options are: warn, info, debug, trace
+      --no-progress           Disable fancy UI progress bars, spinners, logos, etc.
+```
+
+### SEE ALSO
+
+* [zarf tools](zarf_tools.md)	 - Collection of additional tools to make airgap easier
+

docs/6-developer-guide/2-testing.md

@@ -31,7 +31,7 @@ APPLIANCE_MODE=true make test-e2e ARCH="[amd64|arm64]"
 go test ./... -v
 
 # Let's say you only want to run one test. You would run:
-test ./... -v -run TestFooBarBaz
+go test ./... -v -run TestFooBarBaz
 ```
 
 :::note
@@ -58,4 +58,4 @@ The tests are run sequentially and the naming convention is set intentionally:
 - 20 is reserved for `zarf init`
 - 21 is reserved for logging tests so they can be removed first (they take the most resources in the cluster)
 - 22 is reserved for tests required the git-server, which is removed at the end of the test
-- 23-99 are for the remaining tests that only require a basic zarf cluster without logging for the git-server
+- 23-99 are for the remaining tests that only require a basic zarf cluster without logging for the git-server

examples/git-data/zarf.yaml

@@ -17,3 +17,5 @@ components:
       - https://github.com/stefanprodan/podinfo.git
         # Clone an azure repo that breaks in go-git and has to fall back to the host git
       - https://[email protected]/me0515/zarf-public-test/_git/zarf-public-test
+        # Clone an azure repo (w/SHA) that breaks in go-git and has to fall back to the host git
+      - https://[email protected]/me0515/zarf-public-test/_git/zarf-public-test@524980951ff16e19dc25232e9aea8fd693989ba6

src/cmd/package.go

@@ -21,7 +21,6 @@ import (
 
 var insecureDeploy bool
 var shasum string
-var zarfImageCache string
 
 var packageCmd = &cobra.Command{
 	Use:     "package",
@@ -46,8 +45,10 @@ var packageCreateCmd = &cobra.Command{
 			baseDir = args[0]
 		}
 
-		if zarfImageCache != config.ZarfDefaultImageCachePath && cachePathClean(zarfImageCache) {
-			config.SetImageCachePath(zarfImageCache)
+		var isCleanPathRegex = regexp.MustCompile(`^[a-zA-Z0-9\_\-\/\.\~]+$`)
+		if !isCleanPathRegex.MatchString(config.CreateOptions.CachePath) {
+			message.Warnf("Invalid characters in Zarf cache path, defaulting to %s", config.ZarfDefaultCachePath)
+			config.CreateOptions.CachePath = config.ZarfDefaultCachePath
 		}
 
 		packager.Create(baseDir)
@@ -175,15 +176,6 @@ func choosePackage(args []string) string {
 	return path
 }
 
-func cachePathClean(cachePath string) bool {
-	var isCleanPath = regexp.MustCompile(`^[a-zA-Z0-9\_\-\/\.\~]+$`).MatchString
-	if !isCleanPath(cachePath) {
-		message.Warnf("Invalid characters in Zarf cache path, defaulting to ~/%s", config.ZarfDefaultImageCachePath)
-		return false
-	}
-	return true
-}
-
 func init() {
 	rootCmd.AddCommand(packageCmd)
 	packageCmd.AddCommand(packageCreateCmd)
@@ -195,7 +187,7 @@ func init() {
 	packageCreateCmd.Flags().BoolVar(&config.CommonOptions.Confirm, "confirm", false, "Confirm package creation without prompting")
 	packageCreateCmd.Flags().StringVar(&config.CommonOptions.TempDirectory, "tmpdir", "", "Specify the temporary directory to use for intermediate files")
 	packageCreateCmd.Flags().StringToStringVar(&config.CommonOptions.SetVariables, "set", map[string]string{}, "Specify package variables to set on the command line (KEY=value)")
-	packageCreateCmd.Flags().StringVar(&zarfImageCache, "zarf-cache", config.ZarfDefaultImageCachePath, "Specify the location of the Zarf image cache")
+	packageCreateCmd.Flags().StringVar(&config.CreateOptions.CachePath, "zarf-cache", config.ZarfDefaultCachePath, "Specify the location of the Zarf artifact cache (images and git repositories)")
 	packageCreateCmd.Flags().StringVarP(&config.CreateOptions.OutputDirectory, "output-directory", "o", "", "Specify the output directory for the created Zarf package")
 	packageCreateCmd.Flags().BoolVar(&config.CreateOptions.SkipSBOM, "skip-sbom", false, "Skip generating SBOM for this package")
 	packageCreateCmd.Flags().BoolVar(&config.CreateOptions.Insecure, "insecure", false, "Allow insecure registry connections when pulling OCI images")

src/cmd/tools.go

@@ -95,13 +95,27 @@ var k9sCmd = &cobra.Command{
 	},
 }
 
+var clearCacheCmd = &cobra.Command{
+	Use:     "clear-cache",
+	Aliases: []string{"c"},
+	Short:   "Clears the configured git and image cache directory",
+	Run: func(cmd *cobra.Command, args []string) {
+		if err := os.RemoveAll(config.CreateOptions.CachePath); err != nil {
+			message.Fatalf("Unable to clear the cache driectory %s: %s", config.CreateOptions.CachePath, err.Error())
+		}
+	},
+}
+
 func init() {
 	rootCmd.AddCommand(toolsCmd)
 	toolsCmd.AddCommand(archiverCmd)
 	toolsCmd.AddCommand(readCredsCmd)
 	toolsCmd.AddCommand(k9sCmd)
 	toolsCmd.AddCommand(registryCmd)
 
+	toolsCmd.AddCommand(clearCacheCmd)
+	clearCacheCmd.Flags().StringVar(&config.CreateOptions.CachePath, "zarf-cache", config.ZarfDefaultCachePath, "Specify the location of the Zarf  artifact cache (images and git repositories)")
+
 	archiverCmd.AddCommand(archiverCompressCmd)
 	archiverCmd.AddCommand(archiverDecompressCmd)
 

src/config/config.go

@@ -41,9 +41,11 @@ const (
 	ZarfConnectAnnotationDescription = "zarf.dev/connect-description"
 	ZarfConnectAnnotationUrl         = "zarf.dev/connect-url"
 
-	ZarfManagedByLabel        = "app.kubernetes.io/managed-by"
-	ZarfCleanupScriptsPath    = "/opt/zarf"
-	ZarfDefaultImageCachePath = ".zarf-image-cache"
+	ZarfManagedByLabel     = "app.kubernetes.io/managed-by"
+	ZarfCleanupScriptsPath = "/opt/zarf"
+
+	ZarfImageCacheDir = "images"
+	ZarfGitCacheDir   = "repos"
 
 	ZarfYAML    = "zarf.yaml"
 	ZarfSBOMDir = "zarf-sbom"
@@ -89,6 +91,8 @@ var (
 	// Timestamp of when the CLI was started
 	operationStartTime  = time.Now().Unix()
 	dataInjectionMarker = ".zarf-injection-%d"
+
+	ZarfDefaultCachePath = filepath.Join("~", ".zarf-cache")
 )
 
 // Timestamp of when the CLI was started
@@ -284,18 +288,11 @@ func BuildConfig(path string) error {
 	return utils.WriteYaml(path, active, 0400)
 }
 
-func SetImageCachePath(cachePath string) {
-	CreateOptions.ImageCachePath = cachePath
-}
-
-func GetImageCachePath() string {
+// GetAbsCachePath gets the absolute cache path for images and git repos.
+func GetAbsCachePath() string {
 	homePath, _ := os.UserHomeDir()
 
-	if CreateOptions.ImageCachePath == "" {
-		return filepath.Join(homePath, ZarfDefaultImageCachePath)
-	}
-
-	return strings.Replace(CreateOptions.ImageCachePath, "~", homePath, 1)
+	return strings.Replace(CreateOptions.CachePath, "~", homePath, 1)
 }
 
 func isCompatibleComponent(component types.ZarfComponent, filterByOS bool) bool {

src/internal/git/clone.go

@@ -0,0 +1,66 @@
+package git
+
+import (
+	"context"
+	"errors"
+
+	"github.com/defenseunicorns/zarf/src/internal/message"
+	"github.com/defenseunicorns/zarf/src/internal/utils"
+	"github.com/go-git/go-git/v5"
+)
+
+// clone performs a `git clone` of a given repo.
+func clone(gitDirectory string, gitURL string, onlyFetchRef bool, spinner *message.Spinner) (*git.Repository, error) {
+	cloneOptions := &git.CloneOptions{
+		URL:        gitURL,
+		Progress:   spinner,
+		RemoteName: onlineRemoteName,
+	}
+
+	if onlyFetchRef {
+		cloneOptions.Tags = git.NoTags
+	}
+
+	gitCred := FindAuthForHost(gitURL)
+
+	// Gracefully handle no git creds on the system (like our CI/CD)
+	if gitCred.Auth.Username != "" {
+		cloneOptions.Auth = &gitCred.Auth
+	}
+
+	// Clone the given repo
+	repo, err := git.PlainClone(gitDirectory, false, cloneOptions)
+
+	if errors.Is(err, git.ErrRepositoryAlreadyExists) {
+		repo, err = git.PlainOpen(gitDirectory)
+
+		if err != nil {
+			return nil, err
+		}
+
+		return repo, git.ErrRepositoryAlreadyExists
+	} else if err != nil {
+		spinner.Debugf("Failed to clone repo: %s", err)
+		message.Infof("Falling back to host git for %s", gitURL)
+
+		// If we can't clone with go-git, fallback to the host clone
+		// Only support "all tags" due to the azure clone url format including a username
+		cmdArgs := []string{"clone", "--origin", onlineRemoteName, gitURL, gitDirectory}
+
+		if onlyFetchRef {
+			cmdArgs = append(cmdArgs, "--no-tags")
+		}
+
+		stdOut, stdErr, err := utils.ExecCommandWithContext(context.TODO(), false, "git", cmdArgs...)
+		spinner.Updatef(stdOut)
+		spinner.Debugf(stdErr)
+
+		if err != nil {
+			return nil, err
+		}
+
+		return git.PlainOpen(gitDirectory)
+	} else {
+		return repo, nil
+	}
+}

src/internal/git/fetch.go

@@ -1,9 +1,12 @@
 package git
 
 import (
+	"context"
+	"errors"
 	"path"
 
 	"github.com/defenseunicorns/zarf/src/internal/message"
+	"github.com/defenseunicorns/zarf/src/internal/utils"
 	"github.com/go-git/go-git/v5"
 	goConfig "github.com/go-git/go-git/v5/config"
 )
@@ -16,9 +19,7 @@ func fetchTag(gitDirectory string, tag string) {
 
 	err := fetch(gitDirectory, refspec)
 
-	if err == git.ErrTagExists {
-		message.Debug("Tag already fetched")
-	} else if err != nil {
+	if err != nil {
 		message.Fatal(err, "Not a valid tag or unable to fetch")
 	}
 }
@@ -36,8 +37,8 @@ func fetchHash(gitDirectory string, hash string) {
 	}
 }
 
-// fetch performs a `git fetch` of _only_ the provided git refspec.
-func fetch(gitDirectory string, refspec goConfig.RefSpec) error {
+// fetch performs a `git fetch` of _only_ the provided git refspec(s).
+func fetch(gitDirectory string, refspecs ...goConfig.RefSpec) error {
 	repo, err := git.PlainOpen(gitDirectory)
 	if err != nil {
 		message.Fatal(err, "Unable to load the git repo")
@@ -51,18 +52,37 @@ func fetch(gitDirectory string, refspec goConfig.RefSpec) error {
 	}
 
 	gitURL := remotes[0].Config().URLs[0]
-	message.Debugf("Attempting to find ref: %s for %s", refspec.String(), gitURL)
+	message.Debugf("Attempting to find ref: %#v for %s", refspecs, gitURL)
 
 	gitCred := FindAuthForHost(gitURL)
 
 	fetchOptions := &git.FetchOptions{
 		RemoteName: onlineRemoteName,
-		RefSpecs:   []goConfig.RefSpec{refspec},
+		RefSpecs:   refspecs,
 	}
 
 	if gitCred.Auth.Username != "" {
 		fetchOptions.Auth = &gitCred.Auth
 	}
 
-	return repo.Fetch(fetchOptions)
+	err = repo.Fetch(fetchOptions)
+
+	if errors.Is(err, git.ErrTagExists) || errors.Is(err, git.NoErrAlreadyUpToDate) {
+		message.Debug("Already fetched requested ref")
+	} else if err != nil {
+		message.Debugf("Failed to fetch repo: %s", err)
+		message.Infof("Falling back to host git for %s", gitURL)
+
+		// If we can't fetch with go-git, fallback to the host fetch
+		// Only support "all tags" due to the azure fetch url format including a username
+		cmdArgs := []string{"fetch", onlineRemoteName}
+		for _, refspec := range refspecs {
+			cmdArgs = append(cmdArgs, refspec.String())
+		}
+		_, _, err := utils.ExecCommandWithContextAndDir(context.TODO(), gitDirectory, false, "git", cmdArgs...)
+
+		return err
+	}
+
+	return nil
 }