Support git repo caching on package create (#785)

## Description

Add the ability to locally cache large git repositories when pulling
them with Zarf

## Related Issue

Fixes #750

## Type of change

- [X] New feature (non-breaking change which adds functionality)

## Checklist before merging

- [x] Tests have been added/updated as necessary (add the `needs-tests`
label)
- [x] Documentation has been updated as necessary (add the `needs-docs`
label)
- [x] An ADR has been written as necessary (add the `needs-adr` label) [
[1](https://github.com/joelparkerhenderson/architecture-decision-record)
[2](https://cognitect.com/blog/2011/11/15/documenting-architecture-decisions)
[3](https://adr.github.io/) ]
- [x] (Optional) Changes have been linted locally with
[golangci-lint](https://github.com/golangci/golangci-lint). (NOTE: We
haven't turned on lint checks in the pipeline yet so linting may be hard
if it shows a lot of lint errors in places that weren't touched by
changes. Thus, linting is optional right now.)

Co-authored-by: Jon Perry <[email protected]>
Co-authored-by: Megamind <[email protected]>

Makefile

@@ -16,7 +16,7 @@ ifneq ($(UNAME_S),Linux)
 	endif
 endif
 
-AGENT_IMAGE ?= zarfdev/agent:a57bb136f21441c66630403412c6f03fc7f9cd49
+AGENT_IMAGE ?= zarfdev/agent:ef08e77c3880ac64c3cc10ecd314be0869f8f70e
 
 CLI_VERSION := $(if $(shell git describe --tags),$(shell git describe --tags),"UnknownVersion")
 BUILD_ARGS := -s -w -X 'github.com/defenseunicorns/zarf/src/config.CLIVersion=$(CLI_VERSION)'

docs/4-user-guide/1-the-zarf-cli/100-cli-commands/zarf_package_create.md

@@ -22,7 +22,7 @@ zarf package create [DIRECTORY] [flags]
       --set stringToString        Specify package variables to set on the command line (KEY=value) (default [])
       --skip-sbom                 Skip generating SBOM for this package
       --tmpdir string             Specify the temporary directory to use for intermediate files
-      --zarf-cache string         Specify the location of the Zarf image cache (default ".zarf-image-cache")
+      --zarf-cache string         Specify the location of the Zarf artifact cache (images and git repositories) (default "~/.zarf-cache")
 ```
 
 ### Options inherited from parent commands

docs/4-user-guide/1-the-zarf-cli/100-cli-commands/zarf_tools.md

@@ -20,6 +20,7 @@ Collection of additional tools to make airgap easier
 
 * [zarf](zarf.md)	 - DevSecOps Airgap Toolkit
 * [zarf tools archiver](zarf_tools_archiver.md)	 - Compress/Decompress tools for Zarf packages
+* [zarf tools clear-cache](zarf_tools_clear-cache.md)	 - Clears the configured git and image cache directory
 * [zarf tools get-git-password](zarf_tools_get-git-password.md)	 - Returns the push user's password for the Git server
 * [zarf tools monitor](zarf_tools_monitor.md)	 - Launch K9s tool for managing K8s clusters
 * [zarf tools registry](zarf_tools_registry.md)	 - Collection of registry commands provided by Crane

docs/4-user-guide/1-the-zarf-cli/100-cli-commands/zarf_tools_clear-cache.md

@@ -0,0 +1,27 @@
+## zarf tools clear-cache
+
+Clears the configured git and image cache directory
+
+```
+zarf tools clear-cache [flags]
+```
+
+### Options
+
+```
+  -h, --help                help for clear-cache
+      --zarf-cache string   Specify the location of the Zarf  artifact cache (images and git repositories) (default "~/.zarf-cache")
+```
+
+### Options inherited from parent commands
+
+```
+  -a, --architecture string   Architecture for OCI images
+  -l, --log-level string      Log level when running Zarf. Valid options are: warn, info, debug, trace
+      --no-progress           Disable fancy UI progress bars, spinners, logos, etc.
+```
+
+### SEE ALSO
+
+* [zarf tools](zarf_tools.md)	 - Collection of additional tools to make airgap easier
+

docs/6-developer-guide/2-testing.md

@@ -31,7 +31,7 @@ APPLIANCE_MODE=true make test-e2e ARCH="[amd64|arm64]"
 go test ./... -v
 
 # Let's say you only want to run one test. You would run:
-test ./... -v -run TestFooBarBaz
+go test ./... -v -run TestFooBarBaz
 ```
 
 :::note
@@ -58,4 +58,4 @@ The tests are run sequentially and the naming convention is set intentionally:
 - 20 is reserved for `zarf init`
 - 21 is reserved for logging tests so they can be removed first (they take the most resources in the cluster)
 - 22 is reserved for tests required the git-server, which is removed at the end of the test
-- 23-99 are for the remaining tests that only require a basic zarf cluster without logging for the git-server
+- 23-99 are for the remaining tests that only require a basic zarf cluster without logging for the git-server

examples/git-data/zarf.yaml

@@ -17,3 +17,5 @@ components:
       - https://github.com/stefanprodan/podinfo.git
         # Clone an azure repo that breaks in go-git and has to fall back to the host git
       - https://[email protected]/me0515/zarf-public-test/_git/zarf-public-test
+        # Clone an azure repo (w/SHA) that breaks in go-git and has to fall back to the host git
+      - https://[email protected]/me0515/zarf-public-test/_git/zarf-public-test@524980951ff16e19dc25232e9aea8fd693989ba6

examples/terraform/README.md

@@ -0,0 +1,17 @@
+# Terraform
+This example demonstrates how to use Zarf to execute Terraform code to create an S3 bucket.
+
+### Assumptions/Prereqs
+- The binaries in the Zarf package are for an M1 Mac only and will need to be changed for other architectures
+- The S3 bucket name will likely need to be changed as S3 bucket names must be globally unique
+- Your machine has a connection to an AWS instance and is authenticated with an AWS account
+
+### Steps
+
+No K8s cluster is necessary, just build with the package with:
+
+`zarf package create`
+
+And execute with:
+
+`zard package deploy <package_name>`

examples/terraform/main.tf

@@ -0,0 +1,17 @@
+terraform {
+  required_providers {
+    aws = {
+      source = "hashicorp/aws"
+      version = "4.33.0"
+    }
+  }
+}
+
+provider "aws" {
+  region = "us-east-1"
+}
+
+resource "aws_s3_bucket" "example-bucket" {
+  # note this bucket name will need to be changed because s3 buckets must be globally unique
+  bucket = "unclegedds-example-bucket-v3"
+}

examples/terraform/zarf.yaml

@@ -0,0 +1,32 @@
+kind: ZarfPackageConfig
+metadata:
+  name: "terraform"
+  description: "Run terraform/terragrunt code"
+
+components:
+  - name: download-terraform
+    required: true
+    scripts:
+      after:
+        - "rm -f terraform"
+        - "./zarf tools archiver decompress tmp/terraform_1.3.1_darwin_arm64.zip ."
+        - "./zarf tools archiver decompress tmp/terraform-provider-aws_4.33.0_darwin_arm64.zip ./tf-plugins/registry.terraform.io/hashicorp/aws/4.33.0/darwin_arm64"
+        - "rm -rf tmp"
+    files:
+      # terraform code
+      - source: main.tf
+        target: main.tf
+      # mac m1 terraform aws provider binary
+      - source: https://releases.hashicorp.com/terraform-provider-aws/4.33.0/terraform-provider-aws_4.33.0_darwin_arm64.zip
+        target: tmp/terraform-provider-aws_4.33.0_darwin_arm64.zip
+      # mac m1 terraform binary
+      - source: https://releases.hashicorp.com/terraform/1.3.1/terraform_1.3.1_darwin_arm64.zip
+        target: tmp/terraform_1.3.1_darwin_arm64.zip
+
+  - name: execute-terraform
+    scripts:
+      timeoutSeconds: 60
+      showOutput: true
+      before:
+        - "./terraform init -plugin-dir=tf-plugins"
+        - "./terraform apply -auto-approve"

src/cmd/package.go

@@ -21,7 +21,6 @@ import (
 
 var insecureDeploy bool
 var shasum string
-var zarfImageCache string
 
 var packageCmd = &cobra.Command{
 	Use:     "package",
@@ -46,8 +45,10 @@ var packageCreateCmd = &cobra.Command{
 			baseDir = args[0]
 		}
 
-		if zarfImageCache != config.ZarfDefaultImageCachePath && cachePathClean(zarfImageCache) {
-			config.SetImageCachePath(zarfImageCache)
+		var isCleanPathRegex = regexp.MustCompile(`^[a-zA-Z0-9\_\-\/\.\~]+$`)
+		if !isCleanPathRegex.MatchString(config.CreateOptions.CachePath) {
+			message.Warnf("Invalid characters in Zarf cache path, defaulting to %s", config.ZarfDefaultCachePath)
+			config.CreateOptions.CachePath = config.ZarfDefaultCachePath
 		}
 
 		packager.Create(baseDir)
@@ -175,15 +176,6 @@ func choosePackage(args []string) string {
 	return path
 }
 
-func cachePathClean(cachePath string) bool {
-	var isCleanPath = regexp.MustCompile(`^[a-zA-Z0-9\_\-\/\.\~]+$`).MatchString
-	if !isCleanPath(cachePath) {
-		message.Warnf("Invalid characters in Zarf cache path, defaulting to ~/%s", config.ZarfDefaultImageCachePath)
-		return false
-	}
-	return true
-}
-
 func init() {
 	rootCmd.AddCommand(packageCmd)
 	packageCmd.AddCommand(packageCreateCmd)
@@ -195,7 +187,7 @@ func init() {
 	packageCreateCmd.Flags().BoolVar(&config.CommonOptions.Confirm, "confirm", false, "Confirm package creation without prompting")
 	packageCreateCmd.Flags().StringVar(&config.CommonOptions.TempDirectory, "tmpdir", "", "Specify the temporary directory to use for intermediate files")
 	packageCreateCmd.Flags().StringToStringVar(&config.CommonOptions.SetVariables, "set", map[string]string{}, "Specify package variables to set on the command line (KEY=value)")
-	packageCreateCmd.Flags().StringVar(&zarfImageCache, "zarf-cache", config.ZarfDefaultImageCachePath, "Specify the location of the Zarf image cache")
+	packageCreateCmd.Flags().StringVar(&config.CreateOptions.CachePath, "zarf-cache", config.ZarfDefaultCachePath, "Specify the location of the Zarf artifact cache (images and git repositories)")
 	packageCreateCmd.Flags().StringVarP(&config.CreateOptions.OutputDirectory, "output-directory", "o", "", "Specify the output directory for the created Zarf package")
 	packageCreateCmd.Flags().BoolVar(&config.CreateOptions.SkipSBOM, "skip-sbom", false, "Skip generating SBOM for this package")
 	packageCreateCmd.Flags().BoolVar(&config.CreateOptions.Insecure, "insecure", false, "Allow insecure registry connections when pulling OCI images")

src/cmd/tools.go

@@ -95,13 +95,27 @@ var k9sCmd = &cobra.Command{
 	},
 }
 
+var clearCacheCmd = &cobra.Command{
+	Use:     "clear-cache",
+	Aliases: []string{"c"},
+	Short:   "Clears the configured git and image cache directory",
+	Run: func(cmd *cobra.Command, args []string) {
+		if err := os.RemoveAll(config.CreateOptions.CachePath); err != nil {
+			message.Fatalf("Unable to clear the cache driectory %s: %s", config.CreateOptions.CachePath, err.Error())
+		}
+	},
+}
+
 func init() {
 	rootCmd.AddCommand(toolsCmd)
 	toolsCmd.AddCommand(archiverCmd)
 	toolsCmd.AddCommand(readCredsCmd)
 	toolsCmd.AddCommand(k9sCmd)
 	toolsCmd.AddCommand(registryCmd)
 
+	toolsCmd.AddCommand(clearCacheCmd)
+	clearCacheCmd.Flags().StringVar(&config.CreateOptions.CachePath, "zarf-cache", config.ZarfDefaultCachePath, "Specify the location of the Zarf  artifact cache (images and git repositories)")
+
 	archiverCmd.AddCommand(archiverCompressCmd)
 	archiverCmd.AddCommand(archiverDecompressCmd)
 

src/config/config.go

@@ -41,9 +41,11 @@ const (
 	ZarfConnectAnnotationDescription = "zarf.dev/connect-description"
 	ZarfConnectAnnotationUrl         = "zarf.dev/connect-url"
 
-	ZarfManagedByLabel        = "app.kubernetes.io/managed-by"
-	ZarfCleanupScriptsPath    = "/opt/zarf"
-	ZarfDefaultImageCachePath = ".zarf-image-cache"
+	ZarfManagedByLabel     = "app.kubernetes.io/managed-by"
+	ZarfCleanupScriptsPath = "/opt/zarf"
+
+	ZarfImageCacheDir = "images"
+	ZarfGitCacheDir   = "repos"
 
 	ZarfYAML    = "zarf.yaml"
 	ZarfSBOMDir = "zarf-sbom"
@@ -89,6 +91,8 @@ var (
 	// Timestamp of when the CLI was started
 	operationStartTime  = time.Now().Unix()
 	dataInjectionMarker = ".zarf-injection-%d"
+
+	ZarfDefaultCachePath = filepath.Join("~", ".zarf-cache")
 )
 
 // Timestamp of when the CLI was started
@@ -284,18 +288,11 @@ func BuildConfig(path string) error {
 	return utils.WriteYaml(path, active, 0400)
 }
 
-func SetImageCachePath(cachePath string) {
-	CreateOptions.ImageCachePath = cachePath
-}
-
-func GetImageCachePath() string {
+// GetAbsCachePath gets the absolute cache path for images and git repos.
+func GetAbsCachePath() string {
 	homePath, _ := os.UserHomeDir()
 
-	if CreateOptions.ImageCachePath == "" {
-		return filepath.Join(homePath, ZarfDefaultImageCachePath)
-	}
-
-	return strings.Replace(CreateOptions.ImageCachePath, "~", homePath, 1)
+	return strings.Replace(CreateOptions.CachePath, "~", homePath, 1)
 }
 
 func isCompatibleComponent(component types.ZarfComponent, filterByOS bool) bool {

src/internal/git/clone.go

@@ -0,0 +1,66 @@
+package git
+
+import (
+	"context"
+	"errors"
+
+	"github.com/defenseunicorns/zarf/src/internal/message"
+	"github.com/defenseunicorns/zarf/src/internal/utils"
+	"github.com/go-git/go-git/v5"
+)
+
+// clone performs a `git clone` of a given repo.
+func clone(gitDirectory string, gitURL string, onlyFetchRef bool, spinner *message.Spinner) (*git.Repository, error) {
+	cloneOptions := &git.CloneOptions{
+		URL:        gitURL,
+		Progress:   spinner,
+		RemoteName: onlineRemoteName,
+	}
+
+	if onlyFetchRef {
+		cloneOptions.Tags = git.NoTags
+	}
+
+	gitCred := FindAuthForHost(gitURL)
+
+	// Gracefully handle no git creds on the system (like our CI/CD)
+	if gitCred.Auth.Username != "" {
+		cloneOptions.Auth = &gitCred.Auth
+	}
+
+	// Clone the given repo
+	repo, err := git.PlainClone(gitDirectory, false, cloneOptions)
+
+	if errors.Is(err, git.ErrRepositoryAlreadyExists) {
+		repo, err = git.PlainOpen(gitDirectory)
+
+		if err != nil {
+			return nil, err
+		}
+
+		return repo, git.ErrRepositoryAlreadyExists
+	} else if err != nil {
+		spinner.Debugf("Failed to clone repo: %s", err)
+		message.Infof("Falling back to host git for %s", gitURL)
+
+		// If we can't clone with go-git, fallback to the host clone
+		// Only support "all tags" due to the azure clone url format including a username
+		cmdArgs := []string{"clone", "--origin", onlineRemoteName, gitURL, gitDirectory}
+
+		if onlyFetchRef {
+			cmdArgs = append(cmdArgs, "--no-tags")
+		}
+
+		stdOut, stdErr, err := utils.ExecCommandWithContext(context.TODO(), false, "git", cmdArgs...)
+		spinner.Updatef(stdOut)
+		spinner.Debugf(stdErr)
+
+		if err != nil {
+			return nil, err
+		}
+
+		return git.PlainOpen(gitDirectory)
+	} else {
+		return repo, nil
+	}
+}