diff --git a/examples/git-data/README.md b/examples/git-data/README.md index 1606de4bf6..6204398fdb 100644 --- a/examples/git-data/README.md +++ b/examples/git-data/README.md @@ -1,6 +1,6 @@ # Git Data -This example shows how to package `git` repositories to be bundled and pushed across the air gap. This package does not deploy anything itself but pushes assets to the specified `git` service to be consumed as desired. Within Zarf, there are two main ways to include `git` repositories as described below. +This example shows how to package `git` repositories within a Zarf package. This package does not deploy anything itself but pushes assets to the specified `git` service to be consumed as desired. Within Zarf, there are a few ways to include `git` repositories (as described below). :::info @@ -8,11 +8,11 @@ To view the example source code, select the `Edit this page` link below the arti ::: -## Tag-Provided Git Repository Clone +## Tag-Based Git Repository Clone -Tag-provided `git` repository cloning is the recommended way of cloning a `git` repository for air-gapped deployments because it wraps meaning around a specific point in git history that can easily be traced back to the online world. Tag-provided clones are defined using the `scheme://host/repo@tag` format as seen in the example of the `defenseunicorns/zarf` repository (`https://github.com/defenseunicorns/zarf.git@v0.15.0`). +Tag-based `git` repository cloning is the **recommended** way of cloning a `git` repository for air-gapped deployments because it wraps meaning around a specific point in git history that can easily be traced back to the online world. Tag-based clones are defined using the `scheme://host/repo@tag` format as seen in the example of the `defenseunicorns/zarf` repository (`https://github.com/defenseunicorns/zarf.git@v0.15.0`). -A tag-provided clone only mirrors the tag defined in the Zarf definition. The tag will be applied on the `git` mirror to the default trunk branch of the repo (i.e. `master`, `main`, or the default when the repo is cloned). +A tag-based clone only mirrors the tag defined in the Zarf definition. The tag will be applied on the `git` mirror to a zarf-specific branch name based on the tag name (e.g. the tag `v0.1.0` will be pushed to the `zarf-ref-v0.1.0` branch). This ensures that this tag will be pushed and received properly by the airgap `git` server. :::note @@ -20,17 +20,15 @@ If you would like to use a protocol scheme other than http/https, you can do so ::: -## SHA-Provided Git Repository Clone +## SHA-Based Git Repository Clone -SHA-provided `git` repository cloning is another supported way of cloning repos in Zarf but is not recommended as it is less readable/understandable than tag cloning. Commit SHAs are defined using the same `scheme://host/repo@shasum` format as seen in the example of the `defenseunicorns/zarf` repository (`https://github.com/defenseunicorns/zarf.git@c74e2e9626da0400e0a41e78319b3054c53a5d4e`). +In addition to tags, Zarf also supports cloning and pushing a specific SHA hash from a `git` repository, but this is **not recommended** as it is less readable/understandable than tag cloning. Commit SHAs are defined using the same `scheme://host/repo@shasum` format as seen in the example of the `defenseunicorns/zarf` repository (`https://github.com/defenseunicorns/zarf.git@c74e2e9626da0400e0a41e78319b3054c53a5d4e`). -A SHA-provided clone only mirrors the SHA hash defined in the Zarf definition. The SHA will be applied on the `git` mirror to the default trunk branch of the repo (i.e. `master`, `main`, or the default when the repo is cloned) as Zarf does with tagging. +A SHA-based clone only mirrors the SHA hash defined in the Zarf definition. The SHA will be applied on the `git` mirror to a zarf-specific branch name based on the SHA hash (e.g. the SHA `c74e2e9626da0400e0a41e78319b3054c53a5d4e` will be pushed to the `zarf-ref-c74e2e9626da0400e0a41e78319b3054c53a5d4e` branch). This ensures that this tag will be pushed and received properly by the airgap `git` server. -:::note - -If you use a SHA hash or a tag that is on a separate branch this will be placed on the default trunk branch on the offline mirror (i.e. `master`, `main`, etc). This may result in conflicts upon updates if this SHA or tag is not in the update branch's history. +## Git Reference-Based Git Repository Clone -::: +If you need even more control, Zarf also supports providing full `git` [refspecs](https://git-scm.com/book/en/v2/Git-Internals-The-Refspec), as seen in `https://repo1.dso.mil/big-bang/bigbang.git@refs/heads/release-1.53.x`. This allows you to pull specific tags or branches by using this standard. The branch name used by zarf on deploy will depend on the kind of ref specified, branches will use the upstream branch name, whereas other refs (namely tags) will use the `zarf-ref-*` branch name. ## Git Repository Full Clone diff --git a/examples/git-data/zarf.yaml b/examples/git-data/zarf.yaml index 28c4e3572b..5d29db2fe1 100644 --- a/examples/git-data/zarf.yaml +++ b/examples/git-data/zarf.yaml @@ -5,26 +5,38 @@ metadata: description: "Demo Zarf loading resources into a gitops service" components: - - name: full-repo + - name: flux-demo required: true images: - ghcr.io/stefanprodan/podinfo:6.0.0 repos: - # Do a full Git Repo Mirror + # Do a full Git Repo Mirror of a flux repo - https://github.com/stefanprodan/podinfo.git - # Clone an azure repo that breaks in go-git and has to fall back to the host git - - https://me0515@dev.azure.com/me0515/zarf-public-test/_git/zarf-public-test + + - name: full-repo + required: true + repos: + # Do a full Git Repo Mirror + - https://github.com/kelseyhightower/nocode.git - name: specific-tag required: true repos: - # Do a tag-provided Git Repo mirror + # Do a tag-provided Git Repo mirror - https://github.com/defenseunicorns/zarf.git@v0.15.0 + # Use the git refspec pattern to get a tag + - https://github.com/defenseunicorns/zarf.git@refs/tags/v0.16.0 + + - name: specific-branch + required: true + repos: + # Do a branch-provided Git Repo mirror + - "https://repo1.dso.mil/big-bang/bigbang.git@refs/heads/release-1.53.x" - name: specific-hash required: true repos: - # Do a commit hash Git Repo mirror + # Do a commit hash Git Repo mirror - https://github.com/defenseunicorns/zarf.git@c74e2e9626da0400e0a41e78319b3054c53a5d4e - # Clone an azure repo (w/SHA) that breaks in go-git and has to fall back to the host git + # Clone an azure repo (w/SHA) that breaks in go-git and has to fall back to the host git - https://me0515@dev.azure.com/me0515/zarf-public-test/_git/zarf-public-test@524980951ff16e19dc25232e9aea8fd693989ba6 diff --git a/src/internal/cluster/tunnel.go b/src/internal/cluster/tunnel.go index c6c5a6a530..ef81123edb 100644 --- a/src/internal/cluster/tunnel.go +++ b/src/internal/cluster/tunnel.go @@ -99,39 +99,39 @@ func (c *Cluster) PrintConnectTable() error { // ServiceInfoFromNodePortURL takes a nodePortURL and parses it to find the service info for connecting to the cluster. The string is expected to follow the following format: // Example nodePortURL: 127.0.0.1:{PORT}. -func ServiceInfoFromNodePortURL(nodePortURL string) *ServiceInfo { +func ServiceInfoFromNodePortURL(nodePortURL string) (*ServiceInfo, error) { // Attempt to parse as normal, if this fails add a scheme to the URL (docker registries don't use schemes) parsedURL, err := url.Parse(nodePortURL) if err != nil { parsedURL, err = url.Parse("scheme://" + nodePortURL) if err != nil { - return nil + return nil, err } } // Match hostname against localhost ip/hostnames hostname := parsedURL.Hostname() if hostname != config.IPV4Localhost && hostname != "localhost" { - return nil + return nil, fmt.Errorf("node port services should be on localhost") } // Get the node port from the nodeportURL. nodePort, err := strconv.Atoi(parsedURL.Port()) if err != nil { - return nil + return nil, err } if nodePort < 30000 || nodePort > 32767 { - return nil + return nil, fmt.Errorf("node port services should use the port range 30000-32767") } kube, err := k8s.NewWithWait(message.Debugf, labels, defaultTimeout) if err != nil { - return nil + return nil, err } services, err := kube.GetServices("") if err != nil { - return nil + return nil, err } for _, svc := range services.Items { @@ -142,43 +142,43 @@ func ServiceInfoFromNodePortURL(nodePortURL string) *ServiceInfo { Namespace: svc.Namespace, Name: svc.Name, Port: int(port.Port), - } + }, nil } } } } - return nil + return nil, fmt.Errorf("no matching node port services found") } // ServiceInfoFromServiceURL takes a serviceURL and parses it to find the service info for connecting to the cluster. The string is expected to follow the following format: // Example serviceURL: http://{SERVICE_NAME}.{NAMESPACE}.svc.cluster.local:{PORT}. -func ServiceInfoFromServiceURL(serviceURL string) *ServiceInfo { +func ServiceInfoFromServiceURL(serviceURL string) (*ServiceInfo, error) { parsedURL, err := url.Parse(serviceURL) if err != nil { - return nil + return nil, err } // Get the remote port from the serviceURL. remotePort, err := strconv.Atoi(parsedURL.Port()) if err != nil { - return nil + return nil, err } // Match hostname against local cluster service format. pattern := regexp.MustCompile(serviceURLPattern) - matches := pattern.FindStringSubmatch(parsedURL.Hostname()) + get, err := utils.MatchRegex(pattern, parsedURL.Hostname()) // If incomplete match, return an error. - if len(matches) != 3 { - return nil + if err != nil { + return nil, err } return &ServiceInfo{ - Namespace: matches[pattern.SubexpIndex("namespace")], - Name: matches[pattern.SubexpIndex("name")], + Namespace: get("namespace"), + Name: get("name"), Port: remotePort, - } + }, nil } // NewTunnel will create a new Tunnel struct. diff --git a/src/internal/packager/git/checkout.go b/src/internal/packager/git/checkout.go index daec0f405a..1a58400848 100644 --- a/src/internal/packager/git/checkout.go +++ b/src/internal/packager/git/checkout.go @@ -14,25 +14,19 @@ import ( ) // CheckoutTag performs a `git checkout` of the provided tag to a detached HEAD. -func (g *Git) CheckoutTag(tag string) { - message.Debugf("git.CheckoutTag(%s)", tag) - +func (g *Git) CheckoutTag(tag string) error { options := &git.CheckoutOptions{ - Branch: plumbing.ReferenceName("refs/tags/" + tag), + Branch: g.parseRef(tag), } - g.checkout(options) + return g.checkout(options) } func (g *Git) checkoutRefAsBranch(ref string, branch plumbing.ReferenceName) error { - var err error - - if isHash(ref) { - err = g.checkoutHashAsBranch(plumbing.NewHash(ref), branch) - } else { - err = g.checkoutTagAsBranch(ref, branch) + if plumbing.IsHash(ref) { + return g.checkoutHashAsBranch(plumbing.NewHash(ref), branch) } - return err + return g.checkoutTagAsBranch(ref, branch) } // checkoutTagAsBranch performs a `git checkout` of the provided tag but rather @@ -60,8 +54,6 @@ func (g *Git) checkoutTagAsBranch(tag string, branch plumbing.ReferenceName) err func (g *Git) checkoutHashAsBranch(hash plumbing.Hash, branch plumbing.ReferenceName) error { message.Debugf("git.checkoutHasAsBranch(%s,%s)", hash.String(), branch.String()) - _ = g.deleteBranchIfExists(branch) - repo, err := git.PlainOpen(g.GitPath) if err != nil { message.Fatal(err, "Not a valid git repo or unable to open") diff --git a/src/internal/packager/git/clone.go b/src/internal/packager/git/clone.go index db64b6fc2c..c190540592 100644 --- a/src/internal/packager/git/clone.go +++ b/src/internal/packager/git/clone.go @@ -6,67 +6,91 @@ package git import ( "context" - "errors" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/exec" "github.com/go-git/go-git/v5" + goConfig "github.com/go-git/go-git/v5/config" + "github.com/go-git/go-git/v5/plumbing" ) // clone performs a `git clone` of a given repo. -func (g *Git) clone(gitDirectory string, gitURL string, onlyFetchRef bool) (*git.Repository, error) { +func (g *Git) clone(gitURL string, ref plumbing.ReferenceName) error { cloneOptions := &git.CloneOptions{ URL: gitURL, Progress: g.Spinner, RemoteName: onlineRemoteName, } - if onlyFetchRef { + // Don't clone all tags if we're cloning a specific tag. + if ref.IsTag() { cloneOptions.Tags = git.NoTags + cloneOptions.ReferenceName = ref } - gitCred := utils.FindAuthForHost(gitURL) + // Use a single branch if we're cloning a specific branch. + if ref.IsBranch() { + cloneOptions.SingleBranch = true + cloneOptions.ReferenceName = ref + } - // Gracefully handle no git creds on the system (like our CI/CD) + // Setup git credentials if we have them, ignore if we don't. + gitCred := utils.FindAuthForHost(gitURL) if gitCred.Auth.Username != "" { cloneOptions.Auth = &gitCred.Auth } - // Clone the given repo - repo, err := git.PlainClone(gitDirectory, false, cloneOptions) - - if errors.Is(err, git.ErrRepositoryAlreadyExists) { - repo, err = git.PlainOpen(gitDirectory) + // Clone the given repo. + repo, err := git.PlainClone(g.GitPath, false, cloneOptions) + if err != nil { + message.Debugf("Failed to clone repo %s: %s", gitURL, err.Error()) + return g.gitCloneFallback(gitURL, ref) + } - if err != nil { - return nil, err + // If we're cloning the whole repo or a commit hash, we need to also fetch the other branches besides the default. + if ref == emptyRef { + fetchOpts := &git.FetchOptions{ + RemoteName: onlineRemoteName, + Progress: g.Spinner, + RefSpecs: []goConfig.RefSpec{"refs/*:refs/*", "HEAD:refs/heads/HEAD"}, + Tags: git.AllTags, + } + if err := repo.Fetch(fetchOpts); err != nil { + return err } + } - return repo, git.ErrRepositoryAlreadyExists - } else if err != nil { - message.Debugf("Failed to clone repo %s: %s", gitURL, err.Error()) - g.Spinner.Updatef("Falling back to host git for %s", gitURL) + return nil +} - // If we can't clone with go-git, fallback to the host clone - // Only support "all tags" due to the azure clone url format including a username - cmdArgs := []string{"clone", "--origin", onlineRemoteName, gitURL, gitDirectory} +// gitCloneFallback is a fallback if go-git fails to clone a repo. +func (g *Git) gitCloneFallback(gitURL string, ref plumbing.ReferenceName) error { + g.Spinner.Updatef("Falling back to host git for %s", gitURL) - if onlyFetchRef { - cmdArgs = append(cmdArgs, "--no-tags") - } + // If we can't clone with go-git, fallback to the host clone + // Only support "all tags" due to the azure clone url format including a username + cmdArgs := []string{"clone", "--origin", onlineRemoteName, gitURL, g.GitPath} - execConfig := exec.Config{ - Stdout: g.Spinner, - Stderr: g.Spinner, - } - _, _, err := exec.CmdWithContext(context.TODO(), execConfig, "git", cmdArgs...) - if err != nil { - return nil, err - } + // Don't clone all tags if we're cloning a specific tag. + if ref.IsTag() { + cmdArgs = append(cmdArgs, "--no-tags") + } - return git.PlainOpen(gitDirectory) - } else { - return repo, nil + // Use a single branch if we're cloning a specific branch. + if ref.IsBranch() { + cmdArgs = append(cmdArgs, "-b", ref.String()) + cmdArgs = append(cmdArgs, "--single-branch") } + + execConfig := exec.Config{ + Stdout: g.Spinner, + Stderr: g.Spinner, + } + _, _, err := exec.CmdWithContext(context.TODO(), execConfig, "git", cmdArgs...) + if err != nil { + return err + } + + return nil } diff --git a/src/internal/packager/git/common.go b/src/internal/packager/git/common.go index f7c3855d53..8da0fbb6b5 100644 --- a/src/internal/packager/git/common.go +++ b/src/internal/packager/git/common.go @@ -5,29 +5,27 @@ package git import ( - "regexp" + "fmt" + "strings" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/types" + "github.com/go-git/go-git/v5/plumbing" ) // Git is the main struct for managing git repositories. type Git struct { + // Server is the git server configuration. Server types.GitServerInfo - + // Spinner is an optional spinner to use for long running operations. Spinner *message.Spinner - - // Target working directory for the git repository + // Target working directory for the git repository. GitPath string } const onlineRemoteName = "online-upstream" const offlineRemoteName = "offline-downstream" -const onlineRemoteRefPrefix = "refs/remotes/" + onlineRemoteName + "/" - -// isHash checks if a string is a valid git hash. -// https://regex101.com/r/jm9bdk/1 -var isHash = regexp.MustCompile(`^[0-9a-f]{40}$`).MatchString +const emptyRef = "" // New creates a new git instance with the provided server config. func New(server types.GitServerInfo) *Git { @@ -43,3 +41,14 @@ func NewWithSpinner(server types.GitServerInfo, spinner *message.Spinner) *Git { Spinner: spinner, } } + +// parseRef parses the provided ref into a ReferenceName if it's not a hash. +func (g *Git) parseRef(r string) plumbing.ReferenceName { + // If not a full ref, assume it's a tag at this point. + if !plumbing.IsHash(r) && !strings.HasPrefix(r, "refs/") { + r = fmt.Sprintf("refs/tags/%s", r) + } + + // Set the reference name to the provided ref. + return plumbing.ReferenceName(r) +} diff --git a/src/internal/packager/git/fetch.go b/src/internal/packager/git/fetch.go deleted file mode 100644 index f876bea0a4..0000000000 --- a/src/internal/packager/git/fetch.go +++ /dev/null @@ -1,99 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: 2021-Present The Zarf Authors - -// Package git contains functions for interacting with git repositories. -package git - -import ( - "context" - "errors" - "fmt" - - "github.com/defenseunicorns/zarf/src/pkg/message" - "github.com/defenseunicorns/zarf/src/pkg/utils" - "github.com/defenseunicorns/zarf/src/pkg/utils/exec" - "github.com/go-git/go-git/v5" - goConfig "github.com/go-git/go-git/v5/config" -) - -// fetchRef performs a `git fetch` of _only_ the provided git reference (tag or hash). -func (g *Git) fetchRef(ref string) error { - if isHash(ref) { - return g.fetchHash(ref) - } - - return g.fetchTag(ref) -} - -// fetchTag performs a `git fetch` of _only_ the provided tag. -func (g *Git) fetchTag(tag string) error { - refSpec := goConfig.RefSpec(fmt.Sprintf("refs/tags/%s:refs/tags/%s", tag, tag)) - fetchOptions := &git.FetchOptions{ - RemoteName: onlineRemoteName, - RefSpecs: []goConfig.RefSpec{refSpec}, - Tags: git.NoTags, - } - - return g.fetch(g.GitPath, fetchOptions) -} - -// fetchHash performs a `git fetch` of _only_ the provided commit hash. -func (g *Git) fetchHash(hash string) error { - refSpec := goConfig.RefSpec(fmt.Sprintf("%s:%s", hash, hash)) - fetchOptions := &git.FetchOptions{ - RemoteName: onlineRemoteName, - RefSpecs: []goConfig.RefSpec{refSpec}, - Tags: git.NoTags, - } - - return g.fetch(g.GitPath, fetchOptions) -} - -// fetch performs a `git fetch` of _only_ the provided git refspec(s) within the fetchOptions. -func (g *Git) fetch(gitDirectory string, fetchOptions *git.FetchOptions) error { - repo, err := git.PlainOpen(gitDirectory) - if err != nil { - return fmt.Errorf("unable to load the git repo: %w", err) - } - - remotes, err := repo.Remotes() - // There should never be no remotes, but it's easier to account for than - // let be a bug later - if err != nil || len(remotes) == 0 { - return fmt.Errorf("unable to identify remotes: %w", err) - } - - gitURL := remotes[0].Config().URLs[0] - message.Debugf("Attempting to find ref: %#v for %s", fetchOptions.RefSpecs, gitURL) - - gitCred := utils.FindAuthForHost(gitURL) - - if gitCred.Auth.Username != "" { - fetchOptions.Auth = &gitCred.Auth - } - - err = repo.Fetch(fetchOptions) - - if errors.Is(err, git.ErrTagExists) || errors.Is(err, git.NoErrAlreadyUpToDate) { - message.Debug("Already fetched requested ref") - } else if err != nil { - message.Debugf("Failed to fetch repo %s: %s", gitURL, err.Error()) - g.Spinner.Updatef("Falling back to host git for %s", gitURL) - - // If we can't fetch with go-git, fallback to the host fetch - // Only support "all tags" due to the azure fetch url format including a username - cmdArgs := []string{"fetch", onlineRemoteName} - for _, refspec := range fetchOptions.RefSpecs { - cmdArgs = append(cmdArgs, refspec.String()) - } - execCfg := exec.Config{ - Dir: gitDirectory, - Stdout: g.Spinner, - Stderr: g.Spinner, - } - _, _, err := exec.CmdWithContext(context.TODO(), execCfg, "git", cmdArgs...) - return err - } - - return nil -} diff --git a/src/internal/packager/git/pull.go b/src/internal/packager/git/pull.go index 378b282bcc..03bf421bf5 100644 --- a/src/internal/packager/git/pull.go +++ b/src/internal/packager/git/pull.go @@ -6,11 +6,11 @@ package git import ( "fmt" + "path" + "strings" "github.com/defenseunicorns/zarf/src/config" - "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/utils" - "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/plumbing" ) @@ -22,8 +22,8 @@ func (g *Git) DownloadRepoToTemp(gitURL string) (path string, err error) { // If downloading to temp, grab all tags since the repo isn't being // packaged anyway, and it saves us from having to fetch the tags - // later if we need them - if err = g.pull(gitURL, path, ""); err != nil { + // later if we need them. + if err = g.Pull(gitURL, path); err != nil { return "", fmt.Errorf("unable to pull the git repo at %s: %w", gitURL, err) } @@ -31,79 +31,48 @@ func (g *Git) DownloadRepoToTemp(gitURL string) (path string, err error) { } // Pull clones or updates a git repository into the target folder. -func (g *Git) Pull(gitURL, targetFolder string) (path string, err error) { - repoName, err := g.TransformURLtoRepoName(gitURL) +func (g *Git) Pull(gitURL, targetFolder string) error { + g.Spinner.Updatef("Processing git repo %s", gitURL) + + // Find the Zarf-specific repo name from the git URL. + get, err := utils.MatchRegex(gitURLRegex, gitURL) if err != nil { - message.Errorf(err, "unable to pull the git repo at %s", gitURL) - return "", err + return fmt.Errorf("unable to parse git url (%s): %w", gitURL, err) } - path = targetFolder + "/" + repoName - g.GitPath = path - err = g.pull(gitURL, path, repoName) - return path, err -} - -// internal pull function that will clone/pull the latest changes from the git repo -func (g *Git) pull(gitURL, targetFolder string, repoName string) error { - g.Spinner.Updatef("Processing git repo %s", gitURL) + // Setup the reference for this repository + refPlain := get("ref") - matches := gitURLRegex.FindStringSubmatch(gitURL) - idx := gitURLRegex.SubexpIndex + var ref plumbing.ReferenceName - if len(matches) == 0 { - // Unable to find a substring match for the regex - return fmt.Errorf("unable to get extract the repoName from the url %s", gitURL) + // Parse the ref from the git URL. + if refPlain != emptyRef { + ref = g.parseRef(refPlain) } - alreadyProcessed := false - onlyFetchRef := matches[idx("atRef")] != "" - gitURLNoRef := fmt.Sprintf("%s%s/%s%s", matches[idx("proto")], matches[idx("hostPath")], matches[idx("repo")], matches[idx("git")]) - repo, err := g.clone(targetFolder, gitURLNoRef, onlyFetchRef) - if err == git.ErrRepositoryAlreadyExists { - // If we enter this block, the user has specified the same repo twice in one component and we should respect the prior changes - // (see the specific-tag-update component in the git-repo-behavior test-package) - message.Debug("Repo already cloned, pulling any specified changes...") - alreadyProcessed = true - } else if err != nil { + // Construct a path unique to this git repo + repoFolder := fmt.Sprintf("%s-%d", get("repo"), utils.GetCRCHash(gitURL)) + g.GitPath = path.Join(targetFolder, repoFolder) + + // Construct the remote URL without the reference + gitURLNoRef := fmt.Sprintf("%s%s/%s%s", get("proto"), get("hostPath"), get("repo"), get("git")) + + // Clone the git repository. + err = g.clone(gitURLNoRef, ref) + if err != nil { return fmt.Errorf("not a valid git repo or unable to clone (%s): %w", gitURL, err) } - if onlyFetchRef { - ref := matches[idx("ref")] - - // Identify the remote trunk branch name - trunkBranchName := plumbing.NewBranchReferenceName("master") - head, err := repo.Head() - - if err != nil { - // No repo head available - g.Spinner.Errorf(err, "Failed to identify repo head. Ref will be pushed to 'master'.") - } else if head.Name().IsBranch() { - // Valid repo head and it is a branch - trunkBranchName = head.Name() - } else { - // Valid repo head but not a branch - g.Spinner.Errorf(nil, "No branch found for this repo head. Ref will be pushed to 'master'.") - } - - // If this repo has already been processed by Zarf don't remove tags, refs and branches - if !alreadyProcessed { - _, err = g.removeLocalTagRefs() - if err != nil { - return fmt.Errorf("unable to remove unneeded local tag refs: %w", err) - } - _, _ = g.removeLocalBranchRefs() - _, _ = g.removeOnlineRemoteRefs() - } - - err = g.fetchRef(ref) - if err != nil { - return fmt.Errorf("not a valid reference or unable to fetch (%s): %#v", ref, err) - } - - err = g.checkoutRefAsBranch(ref, trunkBranchName) - return err + if ref != emptyRef && !ref.IsBranch() { + // Remove the "refs/tags/" prefix from the ref. + stripped := strings.TrimPrefix(refPlain, "refs/tags/") + + // Use the plain ref as part of the branch name so it is unique and doesn't conflict with other refs. + alias := fmt.Sprintf("zarf-ref-%s", stripped) + trunkBranchName := plumbing.NewBranchReferenceName(alias) + + // Checkout the ref as a branch. + return g.checkoutRefAsBranch(stripped, trunkBranchName) } return nil diff --git a/src/internal/packager/git/push.go b/src/internal/packager/git/push.go index c35c555a90..2d457d7e6b 100644 --- a/src/internal/packager/git/push.go +++ b/src/internal/packager/git/push.go @@ -7,9 +7,11 @@ package git import ( "errors" "fmt" - "path/filepath" + "os" + "path" "github.com/defenseunicorns/zarf/src/pkg/message" + "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/go-git/go-git/v5" goConfig "github.com/go-git/go-git/v5/config" "github.com/go-git/go-git/v5/plumbing/transport" @@ -17,13 +19,31 @@ import ( ) // PushRepo pushes a git repository from the local path to the configured git server. -func (g *Git) PushRepo(localPath string) error { - spinner := message.NewProgressSpinner("Processing git repo at %s", localPath) +func (g *Git) PushRepo(srcUrl, targetFolder string) error { + spinner := message.NewProgressSpinner("Processing git repo %s", srcUrl) defer spinner.Stop() - g.GitPath = localPath - basename := filepath.Base(localPath) - spinner.Updatef("Pushing git repo %s", basename) + // Parse the git URL. + get, err := utils.MatchRegex(gitURLRegex, srcUrl) + if err != nil { + return fmt.Errorf("unable to parse git url (%s): %w", srcUrl, err) + } + + // Setup git paths, including a unique name for the repo based on the hash of the git URL to avoid conflicts. + repoFolder := fmt.Sprintf("%s-%d", get("repo"), utils.GetCRCHash(srcUrl)) + repoPath := path.Join(targetFolder, repoFolder) + + // Check that this package is using the new repo format (if not fallback to the format from <= 0.24.x) + _, err = os.Stat(repoPath) + if os.IsNotExist(err) { + repoFolder, err = g.TransformURLtoRepoName(srcUrl) + if err != nil { + return fmt.Errorf("unable to parse git url (%s): %w", srcUrl, err) + } + repoPath = path.Join(targetFolder, repoFolder) + } + + g.GitPath = repoPath repo, err := g.prepRepoForPush() if err != nil { @@ -32,7 +52,7 @@ func (g *Git) PushRepo(localPath string) error { } if err := g.push(repo, spinner); err != nil { - spinner.Warnf("Unable to push the git repo %s (%s). Retrying....", basename, err.Error()) + spinner.Warnf("Unable to push the git repo %s (%s). Retrying....", get("repo"), err.Error()) return err } @@ -101,27 +121,18 @@ func (g *Git) push(repo *git.Repository, spinner *message.Spinner) error { Password: g.Server.PushPassword, } - // Since we are pushing HEAD:refs/heads/master on deployment, leaving - // duplicates of the HEAD ref (ex. refs/heads/master, - // refs/remotes/online-upstream/master, will cause the push to fail) - removedRefs, err := g.removeHeadCopies() - if err != nil { - return fmt.Errorf("unable to remove unused git refs from the repo: %w", err) - } - // Fetch remote offline refs in case of old update or if multiple refs are specified in one package fetchOptions := &git.FetchOptions{ RemoteName: offlineRemoteName, Auth: &gitCred, RefSpecs: []goConfig.RefSpec{ "refs/heads/*:refs/heads/*", - onlineRemoteRefPrefix + "*:refs/heads/*", "refs/tags/*:refs/tags/*", }, } // Attempt the fetch, if it fails, log a warning and continue trying to push (might as well try..) - err = repo.Fetch(fetchOptions) + err := repo.Fetch(fetchOptions) if errors.Is(err, transport.ErrRepositoryNotFound) { message.Debugf("Repo not yet available offline, skipping fetch...") } else if errors.Is(err, git.ErrForceNeeded) { @@ -137,10 +148,11 @@ func (g *Git) push(repo *git.Repository, spinner *message.Spinner) error { RemoteName: offlineRemoteName, Auth: &gitCred, Progress: spinner, + // TODO: (@JEFFMCCOY) add the parsing for the `+` force prefix (see https://github.com/defenseunicorns/zarf/issues/1410) + //Force: isForce, // If a provided refspec doesn't push anything, it is just ignored RefSpecs: []goConfig.RefSpec{ "refs/heads/*:refs/heads/*", - onlineRemoteRefPrefix + "*:refs/heads/*", "refs/tags/*:refs/tags/*", }, }) @@ -151,9 +163,5 @@ func (g *Git) push(repo *git.Repository, spinner *message.Spinner) error { return fmt.Errorf("unable to push repo to the gitops service: %w", err) } - // Add back the refs we removed just incase this push isn't the last thing - // being run and a later task needs to reference them. - g.addRefs(removedRefs) - return nil } diff --git a/src/internal/packager/git/refs.go b/src/internal/packager/git/refs.go deleted file mode 100644 index 6425e34656..0000000000 --- a/src/internal/packager/git/refs.go +++ /dev/null @@ -1,169 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: 2021-Present The Zarf Authors - -// Package git contains functions for interacting with git repositories. -package git - -import ( - "fmt" - "strings" - - "github.com/defenseunicorns/zarf/src/pkg/message" - "github.com/go-git/go-git/v5" - "github.com/go-git/go-git/v5/plumbing" -) - -// removeLocalBranchRefs removes all refs that are local branches -// It returns a slice of references deleted. -func (g *Git) removeLocalBranchRefs() ([]*plumbing.Reference, error) { - return g.removeReferences( - func(ref *plumbing.Reference) bool { - return ref.Name().IsBranch() - }, - ) -} - -// removeLocalTagRefs removes all tags in the local repo. -// It returns a slice of tags deleted. -func (g *Git) removeLocalTagRefs() ([]string, error) { - removedTags := []string{} - repo, err := git.PlainOpen(g.GitPath) - if err != nil { - return removedTags, fmt.Errorf("not a valid git repo or unable to open: %w", err) - } - - allTags, err := repo.Tags() - if err != nil { - return removedTags, fmt.Errorf("failed to get the tags for the repo: %w", err) - } - - err = allTags.ForEach(func(t *plumbing.Reference) error { - removedTags = append(removedTags, t.Name().Short()) - tagErr := repo.DeleteTag(string(t.Name().Short())) - if tagErr != nil { - return fmt.Errorf("failed to delete tag %s: %w", t.Name().Short(), tagErr) - } - return nil - }) - - return removedTags, err -} - -// removeOnlineRemoteRefs removes all refs pointing to the online-upstream -// It returns a slice of references deleted. -func (g *Git) removeOnlineRemoteRefs() ([]*plumbing.Reference, error) { - return g.removeReferences( - func(ref *plumbing.Reference) bool { - return strings.HasPrefix(ref.Name().String(), onlineRemoteRefPrefix) - }, - ) -} - -// removeHeadCopies removes any refs that aren't HEAD but have the same hash -// It returns a slice of references deleted. -func (g *Git) removeHeadCopies() ([]*plumbing.Reference, error) { - message.Debugf("git.removeHeadCopies()") - - repo, err := git.PlainOpen(g.GitPath) - if err != nil { - return nil, fmt.Errorf("not a valid git repo or unable to open: %w", err) - } - - head, err := repo.Head() - if err != nil { - return nil, fmt.Errorf("failed to identify references when getting the repo's head: %w", err) - } - - headHash := head.Hash().String() - return g.removeReferences( - func(ref *plumbing.Reference) bool { - // Don't ever remove tags - return !ref.Name().IsTag() && ref.Hash().String() == headHash - }, - ) -} - -// removeReferences removes references based on a provided callback -// removeReferences does not allow you to delete HEAD -// It returns a slice of references deleted. -func (g *Git) removeReferences(shouldRemove func(*plumbing.Reference) bool) ([]*plumbing.Reference, error) { - message.Debugf("git.removeReferences()") - repo, err := git.PlainOpen(g.GitPath) - if err != nil { - return nil, fmt.Errorf("not a valid git repo or unable to open: %w", err) - } - - references, err := repo.References() - if err != nil { - return nil, fmt.Errorf("failed to identify references when getting the repo's references: %w", err) - } - - head, err := repo.Head() - if err != nil { - return nil, fmt.Errorf("failed to identify head: %w", err) - } - - var removedRefs []*plumbing.Reference - err = references.ForEach(func(ref *plumbing.Reference) error { - refIsNotHeadOrHeadTarget := ref.Name() != plumbing.HEAD && ref.Name() != head.Name() - // Run shouldRemove inline here to take advantage of short circuit - // evaluation as to not waste a cycle on HEAD - if refIsNotHeadOrHeadTarget && shouldRemove(ref) { - err = repo.Storer.RemoveReference(ref.Name()) - if err != nil { - return err - } - removedRefs = append(removedRefs, ref) - } - return nil - }) - - if err != nil { - return nil, fmt.Errorf("failed to remove references: %w", err) - } - - return removedRefs, nil -} - -// addRefs adds a provided arbitrary list of references to a repo -// It is intended to be used with references returned by a Remove function. -func (g *Git) addRefs(refs []*plumbing.Reference) error { - message.Debugf("git.addRefs()") - repo, err := git.PlainOpen(g.GitPath) - if err != nil { - return fmt.Errorf("not a valid git repo or unable to open: %w", err) - } - - for _, ref := range refs { - err = repo.Storer.SetReference(ref) - if err != nil { - return fmt.Errorf("failed to add references: %w", err) - } - } - - return nil -} - -// deleteBranchIfExists ensures the provided branch name does not exist. -func (g *Git) deleteBranchIfExists(branchName plumbing.ReferenceName) error { - message.Debugf("g.deleteBranchIfExists(%s)", branchName.String()) - - repo, err := git.PlainOpen(g.GitPath) - if err != nil { - return fmt.Errorf("not a valid git repo or unable to open: %w", err) - } - - // Deletes the branch by name - err = repo.DeleteBranch(branchName.Short()) - if err != nil && err != git.ErrBranchNotFound { - return fmt.Errorf("failed to delete branch: %w", err) - } - - // Delete reference too - err = repo.Storer.RemoveReference(branchName) - if err != nil && err != git.ErrInvalidReference { - return fmt.Errorf("failed to delete branch reference: %w", err) - } - - return nil -} diff --git a/src/internal/packager/git/url.go b/src/internal/packager/git/url.go index b4c17f9e17..9ed957c1a4 100644 --- a/src/internal/packager/git/url.go +++ b/src/internal/packager/git/url.go @@ -6,14 +6,14 @@ package git import ( "fmt" - "hash/crc32" "regexp" "github.com/defenseunicorns/zarf/src/pkg/message" + "github.com/defenseunicorns/zarf/src/pkg/utils" ) -// For further explanation: https://regex101.com/r/zq64q4/1. -var gitURLRegex = regexp.MustCompile(`^(?P[a-z]+:\/\/)(?P.+?)\/(?P[\w\-\.]+?)(?P\.git)?(?P@(?P[\w\-\.]+))?$`) +// For further explanation: https://regex101.com/r/xx8NQe/1. +var gitURLRegex = regexp.MustCompile(`^(?P[a-z]+:\/\/)(?P.+?)\/(?P[\w\-\.]+?)(?P\.git)?(?P@(?P\+)?(?P[\/\+\w\-\.]+))?$`) // MutateGitURLsInText changes the gitURL hostname to use the repository Zarf is configured to use. func (g *Git) MutateGitURLsInText(text string) string { @@ -30,22 +30,21 @@ func (g *Git) MutateGitURLsInText(text string) string { // TransformURLtoRepoName takes a git url and returns a Zarf-compatible repo name. func (g *Git) TransformURLtoRepoName(url string) (string, error) { - matches := gitURLRegex.FindStringSubmatch(url) - idx := gitURLRegex.SubexpIndex + get, err := utils.MatchRegex(gitURLRegex, url) - if len(matches) == 0 { + if err != nil { // Unable to find a substring match for the regex return "", fmt.Errorf("unable to get extract the repoName from the url %s", url) } - repoName := matches[idx("repo")] + repoName := get("repo") // NOTE: We remove the .git and protocol so that https://zarf.dev/repo.git and http://zarf.dev/repo // resolve to the same repp (as they would in real life) - sanitizedURL := fmt.Sprintf("%s/%s", matches[idx("hostPath")], repoName) + sanitizedURL := fmt.Sprintf("%s/%s", get("hostPath"), repoName) // Add crc32 hash of the repoName to the end of the repo - table := crc32.MakeTable(crc32.IEEE) - checksum := crc32.Checksum([]byte(sanitizedURL), table) + checksum := utils.GetCRCHash(sanitizedURL) + newRepoName := fmt.Sprintf("%s-%d", repoName, checksum) return newRepoName, nil diff --git a/src/internal/packager/images/push.go b/src/internal/packager/images/push.go index 28f1cc7d7a..214b4afaab 100644 --- a/src/internal/packager/images/push.go +++ b/src/internal/packager/images/push.go @@ -24,7 +24,6 @@ func (i *ImgConfig) PushToZarfRegistry() error { target string ) - registryURL = i.RegInfo.Address if i.RegInfo.InternalRegistry { // Establish a registry tunnel to send the images to the zarf registry if tunnel, err = cluster.NewZarfTunnel(); err != nil { @@ -32,9 +31,10 @@ func (i *ImgConfig) PushToZarfRegistry() error { } target = cluster.ZarfRegistry } else { - svcInfo := cluster.ServiceInfoFromNodePortURL(i.RegInfo.Address) - if svcInfo != nil { - // If this is a service, create a port-forward tunnel to that resource + svcInfo, err := cluster.ServiceInfoFromNodePortURL(i.RegInfo.Address) + + // If this is a service (no error getting svcInfo), create a port-forward tunnel to that resource + if err == nil { if tunnel, err = cluster.NewTunnel(svcInfo.Namespace, cluster.SvcResource, svcInfo.Name, 0, svcInfo.Port); err != nil { return err } diff --git a/src/pkg/packager/actions.go b/src/pkg/packager/actions.go index 82249ebed6..a6f9c0b85d 100644 --- a/src/pkg/packager/actions.go +++ b/src/pkg/packager/actions.go @@ -15,6 +15,7 @@ import ( "github.com/defenseunicorns/zarf/src/internal/packager/template" "github.com/defenseunicorns/zarf/src/pkg/message" + "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/exec" "github.com/defenseunicorns/zarf/src/types" ) @@ -211,10 +212,9 @@ func actionCmdMutation(cmd string) (string, error) { // Convert any ${ZARF_VAR_*} or $ZARF_VAR_* to ${env:ZARF_VAR_*} or $env:ZARF_VAR_* respectively (also TF_VAR_*). // https://regex101.com/r/xk1rkw/1 envVarRegex := regexp.MustCompile(`(?P\${?(?P(ZARF|TF)_VAR_([a-zA-Z0-9_-])+)}?)`) - matches := envVarRegex.FindStringSubmatch(cmd) - matchIndex := envVarRegex.SubexpIndex - if len(matches) > 0 { - newCmd := strings.ReplaceAll(cmd, matches[matchIndex("envIndicator")], fmt.Sprintf("$Env:%s", matches[matchIndex("varName")])) + get, err := utils.MatchRegex(envVarRegex, cmd) + if err == nil { + newCmd := strings.ReplaceAll(cmd, get("envIndicator"), fmt.Sprintf("$Env:%s", get("varName"))) message.Debugf("Converted command \"%s\" to \"%s\" t", cmd, newCmd) cmd = newCmd } diff --git a/src/pkg/packager/create.go b/src/pkg/packager/create.go index d4641844f8..fe93835419 100644 --- a/src/pkg/packager/create.go +++ b/src/pkg/packager/create.go @@ -391,7 +391,7 @@ func (p *Packager) addComponent(component types.ZarfComponent) (*types.Component for _, url := range component.Repos { // Pull all the references if there is no `@` in the string gitCfg := git.NewWithSpinner(p.cfg.State.GitServer, spinner) - if _, err := gitCfg.Pull(url, componentPath.Repos); err != nil { + if err := gitCfg.Pull(url, componentPath.Repos); err != nil { return nil, fmt.Errorf("unable to pull git repo %s: %w", url, err) } } diff --git a/src/pkg/packager/deploy.go b/src/pkg/packager/deploy.go index b9edcb263a..d61bd288e7 100644 --- a/src/pkg/packager/deploy.go +++ b/src/pkg/packager/deploy.go @@ -406,12 +406,11 @@ func (p *Packager) pushReposToRepository(reposPath string, repos []string) error // Create an anonymous function to push the repo to the Zarf git server tryPush := func() error { gitClient := git.New(p.cfg.State.GitServer) + svcInfo, err := cluster.ServiceInfoFromServiceURL(gitClient.Server.Address) - svcInfo := cluster.ServiceInfoFromServiceURL(gitClient.Server.Address) - // If this is a service, create a port-forward tunnel to that resource - if svcInfo != nil { + // If this is a service (no error getting svcInfo), create a port-forward tunnel to that resource + if err == nil { tunnel, err := cluster.NewTunnel(svcInfo.Namespace, cluster.SvcResource, svcInfo.Name, 0, svcInfo.Port) - if err != nil { return err } @@ -421,13 +420,7 @@ func (p *Packager) pushReposToRepository(reposPath string, repos []string) error gitClient.Server.Address = tunnel.HTTPEndpoint() } - // Convert the repo URL to a Zarf-formatted repo name - repoPath, err := gitClient.TransformURLtoRepoName(repoURL) - if err != nil { - return fmt.Errorf("unable to get the repo name from the URL %s: %w", repoURL, err) - } - - return gitClient.PushRepo(filepath.Join(reposPath, repoPath)) + return gitClient.PushRepo(repoURL, reposPath) } // Try repo push up to 3 times diff --git a/src/pkg/utils/hash.go b/src/pkg/utils/hash.go index 65e3520127..af5e4bd4c9 100644 --- a/src/pkg/utils/hash.go +++ b/src/pkg/utils/hash.go @@ -7,6 +7,7 @@ package utils import ( "crypto" "encoding/hex" + "hash/crc32" "io" "os" @@ -39,3 +40,9 @@ func GetCryptoHash(path string, hashName crypto.Hash) (string, error) { return hex.EncodeToString(hash.Sum(nil)), nil } + +// GetCRCHash returns the computed CRC32 Sum of a given string +func GetCRCHash(text string) uint32 { + table := crc32.MakeTable(crc32.IEEE) + return crc32.Checksum([]byte(text), table) +} diff --git a/src/pkg/utils/image.go b/src/pkg/utils/image.go index 94ed871e66..3441be714c 100644 --- a/src/pkg/utils/image.go +++ b/src/pkg/utils/image.go @@ -6,7 +6,6 @@ package utils import ( "fmt" - "hash/crc32" "github.com/distribution/distribution/reference" ) @@ -30,8 +29,7 @@ func SwapHost(src string, targetHost string) (string, error) { } // Generate a crc32 hash of the image host + name - table := crc32.MakeTable(crc32.IEEE) - checksum := crc32.Checksum([]byte(image.Name), table) + checksum := GetCRCHash(image.Name) return fmt.Sprintf("%s/%s-%d%s", targetHost, image.Path, checksum, image.TagOrDigest), nil } diff --git a/src/pkg/utils/misc.go b/src/pkg/utils/misc.go index 79e9c060b6..b972976a32 100644 --- a/src/pkg/utils/misc.go +++ b/src/pkg/utils/misc.go @@ -5,10 +5,12 @@ package utils import ( + "fmt" + "regexp" "time" ) -// Unique returns a new slice with only unique elements +// Unique returns a new slice with only unique elements. func Unique[T comparable](s []T) (r []T) { exists := make(map[T]bool) for _, str := range s { @@ -20,7 +22,7 @@ func Unique[T comparable](s []T) (r []T) { return r } -// Reverse returns a new slice with the elements in reverse order +// Reverse returns a new slice with the elements in reverse order. func Reverse[T any](s []T) (r []T) { for i := len(s) - 1; i >= 0; i-- { r = append(r, s[i]) @@ -28,7 +30,7 @@ func Reverse[T any](s []T) (r []T) { return r } -// Filter returns a new slice with only the elements that pass the test +// Filter returns a new slice with only the elements that pass the test. func Filter[T any](ss []T, test func(T) bool) (r []T) { for _, s := range ss { if test(s) { @@ -38,7 +40,7 @@ func Filter[T any](ss []T, test func(T) bool) (r []T) { return r } -// Find returns the first element that passes the test +// Find returns the first element that passes the test. func Find[T any](ss []T, test func(T) bool) (r T) { for _, s := range ss { if test(s) { @@ -48,7 +50,7 @@ func Find[T any](ss []T, test func(T) bool) (r T) { return r } -// RemoveMatches removes the given element from the slice that matches the test +// RemoveMatches removes the given element from the slice that matches the test. func RemoveMatches[T any](ss []T, test func(T) bool) (r []T) { for _, s := range ss { if !test(s) { @@ -58,7 +60,7 @@ func RemoveMatches[T any](ss []T, test func(T) bool) (r []T) { return r } -// Retry will retry a function until it succeeds or the timeout is reached, timeout == retries * delay +// Retry will retry a function until it succeeds or the timeout is reached, timeout == retries * delay. func Retry(fn func() error, retries int, delay time.Duration) (err error) { for r := 0; r < retries; r++ { err = fn() @@ -72,7 +74,7 @@ func Retry(fn func() error, retries int, delay time.Duration) (err error) { return err } -// SliceContains returns true if the given element is in the slice +// SliceContains returns true if the given element is in the slice. func SliceContains[T comparable](s []T, e T) bool { for _, v := range s { if v == e { @@ -82,7 +84,7 @@ func SliceContains[T comparable](s []T, e T) bool { return false } -// MergeMap merges map m2 with m1 overwriting common values with m2's values +// MergeMap merges map m2 with m1 overwriting common values with m2's values. func MergeMap[T any](m1 map[string]T, m2 map[string]T) (r map[string]T) { r = map[string]T{} @@ -97,7 +99,7 @@ func MergeMap[T any](m1 map[string]T, m2 map[string]T) (r map[string]T) { return r } -// TransformMapKeys takes a map and transforms its keys using the provided function +// TransformMapKeys takes a map and transforms its keys using the provided function. func TransformMapKeys[T any](m map[string]T, transform func(string) string) (r map[string]T) { r = map[string]T{} @@ -107,3 +109,20 @@ func TransformMapKeys[T any](m map[string]T, transform func(string) string) (r m return r } + +// MatchRegex wraps a get function around a substring match. +func MatchRegex(regex *regexp.Regexp, str string) (func(string) string, error) { + // Validate the string. + matches := regex.FindStringSubmatch(str) + + // Parse the string into its components. + get := func(name string) string { + return matches[regex.SubexpIndex(name)] + } + + if len(matches) == 0 { + return get, fmt.Errorf("unable to match against %s", str) + } + + return get, nil +} diff --git a/src/test/e2e/07_create_git_test.go b/src/test/e2e/07_create_git_test.go index 7a7072a21c..110894e125 100644 --- a/src/test/e2e/07_create_git_test.go +++ b/src/test/e2e/07_create_git_test.go @@ -7,46 +7,69 @@ package test import ( "fmt" "os" - "os/exec" "path/filepath" "testing" + "github.com/defenseunicorns/zarf/src/pkg/utils/exec" "github.com/stretchr/testify/require" ) func TestCreateGit(t *testing.T) { + t.Log("E2E: Test Git Repo Behavior") + extractDir := filepath.Join(os.TempDir(), ".extracted-git-pkg") + e2e.cleanFiles(extractDir) - pkgDir := "src/test/test-packages/git-repo-behavior" - pkgPath := fmt.Sprintf("%s/zarf-package-git-behavior-%s.tar.zst", pkgDir, e2e.arch) - outputFlag := fmt.Sprintf("-o=%s", pkgDir) - e2e.cleanFiles(extractDir, pkgPath) + // Extract the test package. + path := fmt.Sprintf("build/zarf-package-git-data-%s-v1.0.0.tar.zst", e2e.arch) + stdOut, stdErr, err := e2e.execZarfCommand("tools", "archiver", "decompress", path, extractDir) + require.NoError(t, err, stdOut, stdErr) + defer e2e.cleanFiles(extractDir) - _, _, err := e2e.execZarfCommand("package", "create", pkgDir, outputFlag, "--confirm") - require.NoError(t, err, "error when building the test package") - // defer e2e.cleanFiles(pkgPath) + // Verify the full-repo component. + gitDirFlag := fmt.Sprintf("--git-dir=%s/components/full-repo/repos/nocode-953829860/.git", extractDir) + stdOut, stdErr, err = exec.Cmd("git", gitDirFlag, "log", "--oneline", "--decorate") + require.NoError(t, err, stdOut, stdErr) + require.Contains(t, stdOut, "c46f06e add no code") + require.Contains(t, stdOut, "(tag: 1.0.0)") + require.Contains(t, stdOut, "(HEAD -> master, online-upstream/master, HEAD)") - stdOut, stdErr, err := e2e.execZarfCommand("tools", "archiver", "decompress", pkgPath, extractDir) + // Verify a repo with a shorthand tag. + gitDirFlag = fmt.Sprintf("--git-dir=%s/components/specific-tag/repos/zarf-4023393304/.git", extractDir) + stdOut, stdErr, err = exec.Cmd("git", gitDirFlag, "log", "HEAD^..HEAD", "--oneline", "--decorate") require.NoError(t, err, stdOut, stdErr) - // defer e2e.cleanFiles(extractDir) + require.Contains(t, stdOut, "9eb207e (HEAD -> zarf-ref-v0.15.0, tag: v0.15.0) Normalize --confirm behavior in the CLI (#297)") - // Verify the main zarf repo only has one tag - gitDirFlag := fmt.Sprintf("--git-dir=%s/components/specific-tag/repos/zarf-1211668992/.git", extractDir) - gitTagOut, err := exec.Command("git", gitDirFlag, "tag", "-l").Output() - require.NoError(t, err) - require.Equal(t, "v0.15.0\n", string(gitTagOut)) + // Verify a repo with a shorthand tag only has one tag. + stdOut, stdErr, err = exec.Cmd("git", gitDirFlag, "tag") + require.NoError(t, err, stdOut, stdErr) + require.Equal(t, "v0.15.0\n", stdOut) - gitHeadOut, err := exec.Command("git", gitDirFlag, "rev-parse", "HEAD").Output() - require.NoError(t, err) - require.Equal(t, "9eb207e552fe3a73a9ced064d35a9d9872dfbe6d\n", string(gitHeadOut)) + // Verify a repo with a full git refspec tag. + gitDirFlag = fmt.Sprintf("--git-dir=%s/components/specific-tag/repos/zarf-2175050463/.git", extractDir) + stdOut, stdErr, err = exec.Cmd("git", gitDirFlag, "log", "HEAD^..HEAD", "--oneline", "--decorate") + require.NoError(t, err, stdOut, stdErr) + require.Contains(t, stdOut, "58e3cd5 (HEAD -> zarf-ref-v0.16.0, tag: v0.16.0) slightly re-arrange zarf arch diagram layout (#383)") - // Verify the second zarf repo only has two tags - gitDirFlag = fmt.Sprintf("--git-dir=%s/components/specific-tag-update/repos/zarf-1211668992/.git", extractDir) - gitTagOut, err = exec.Command("git", gitDirFlag, "tag", "-l").Output() - require.NoError(t, err) - require.Equal(t, "v0.16.0\nv0.17.0\n", string(gitTagOut)) + // Verify a repo with a full git refspec tag only has one tag. + stdOut, stdErr, err = exec.Cmd("git", gitDirFlag, "tag") + require.NoError(t, err, stdOut, stdErr) + require.Equal(t, "v0.16.0\n", stdOut) - gitHeadOut, err = exec.Command("git", gitDirFlag, "rev-parse", "HEAD").Output() - require.NoError(t, err) - require.Equal(t, "bea100213565de1348375828e14be6e1482a67f8\n", string(gitHeadOut)) + // Verify a repo with a branch. + gitDirFlag = fmt.Sprintf("--git-dir=%s/components/specific-branch/repos/bigbang-3067531188/.git", extractDir) + stdOut, stdErr, err = exec.Cmd("git", gitDirFlag, "log", "HEAD^..HEAD", "--oneline", "--decorate") + require.NoError(t, err, stdOut, stdErr) + require.Contains(t, stdOut, "ab6407fc (HEAD -> release-1.53.x, tag: 1.53.0-rc.1, tag: 1.53.0, online-upstream/release-1.53.x)") + + // Verify a repo with a branch only has one branch. + stdOut, stdErr, err = exec.Cmd("git", gitDirFlag, "branch") + require.NoError(t, err, stdOut, stdErr) + require.Equal(t, "* release-1.53.x\n", stdOut) + + // Verify a repo with a commit hash. + gitDirFlag = fmt.Sprintf("--git-dir=%s/components/specific-hash/repos/zarf-1356873667/.git", extractDir) + stdOut, stdErr, err = exec.Cmd("git", gitDirFlag, "log", "HEAD^..HEAD", "--oneline", "--decorate") + require.NoError(t, err, stdOut, stdErr) + require.Contains(t, stdOut, "c74e2e9 (HEAD -> zarf-ref-c74e2e9626da0400e0a41e78319b3054c53a5d4e, tag: v0.21.3) Re-add docker buildx for release pipeilne") } diff --git a/src/test/e2e/22_git_and_flux_test.go b/src/test/e2e/22_git_and_flux_test.go index ab22a3458a..de1ee32f21 100644 --- a/src/test/e2e/22_git_and_flux_test.go +++ b/src/test/e2e/22_git_and_flux_test.go @@ -95,14 +95,10 @@ func testGitServerTagAndHash(t *testing.T, gitURL string) { // Get the Zarf repo commit repoHash := "c74e2e9626da0400e0a41e78319b3054c53a5d4e" - getRepoCommitsRequest, _ := http.NewRequest("GET", fmt.Sprintf("%s/api/v1/repos/%s/%s/commits", gitURL, config.ZarfGitPushUser, repoName), nil) + getRepoCommitsRequest, _ := http.NewRequest("GET", fmt.Sprintf("%s/api/v1/repos/%s/%s/git/commits/%s", gitURL, config.ZarfGitPushUser, repoName, repoHash), nil) getRepoCommitsResponseBody, err := gitCfg.DoHTTPThings(getRepoCommitsRequest, config.ZarfGitReadUser, state.GitServer.PullPassword) assert.NoError(t, err) - - // Make sure the pushed commit exists - var commitMap []map[string]interface{} - json.Unmarshal(getRepoCommitsResponseBody, &commitMap) - assert.Equal(t, repoHash, commitMap[0]["sha"]) + assert.Contains(t, string(getRepoCommitsResponseBody), repoHash) } func waitFluxPodInfoDeployment(t *testing.T) { diff --git a/src/test/test-packages/git-repo-behavior/zarf.yaml b/src/test/test-packages/git-repo-behavior/zarf.yaml deleted file mode 100644 index c6adef4b44..0000000000 --- a/src/test/test-packages/git-repo-behavior/zarf.yaml +++ /dev/null @@ -1,27 +0,0 @@ -kind: ZarfPackageConfig -metadata: - name: git-behavior - description: "Demo Zarf loading resources into a gitops service" - -components: - - name: specific-tag - required: true - repos: - # Do a tag-provided Git Repo mirror - - https://github.com/defenseunicorns/zarf.git@v0.15.0 - - - name: specific-tag-update - required: true - repos: - # Do a tag-provided Git Repo mirror - - https://github.com/defenseunicorns/zarf.git@v0.16.0 - # Do a tag-provided Git Repo mirror - - https://github.com/defenseunicorns/zarf.git@v0.17.0 - - - name: specific-hash - required: true - repos: - # Do a commit hash Git Repo mirror - - https://github.com/defenseunicorns/zarf.git@c74e2e9626da0400e0a41e78319b3054c53a5d4e - # Clone an azure repo (w/SHA) that breaks in go-git and has to fall back to the host git - - https://me0515@dev.azure.com/me0515/zarf-public-test/_git/zarf-public-test@524980951ff16e19dc25232e9aea8fd693989ba6