From e880a45474b3fffdf6d275deff0df1a63deec1e6 Mon Sep 17 00:00:00 2001 From: Hidde Beydals Date: Wed, 13 May 2020 18:29:47 +0200 Subject: [PATCH] controllers: control tar ignores w/ exclude files This commit changes the file excludes for tarballs generated for Git repository artifacts from a fixed set of strings to include exclusion files files. It currently takes `.sourceignore` and in the root of the given directory into account. In addition to this the Git VCS related files that are ignored have been extended to not only include the .git/ directory, but also the .gitignore, .gitmodules and .gitattributes files. Mimicking part of the --exclude-vcs flag not available on all tar versions. --- controllers/gitrepository_controller.go | 2 +- controllers/storage.go | 27 +++++++++++++++++-------- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/controllers/gitrepository_controller.go b/controllers/gitrepository_controller.go index b8179af00..f987e8d5d 100644 --- a/controllers/gitrepository_controller.go +++ b/controllers/gitrepository_controller.go @@ -345,7 +345,7 @@ func (r *GitRepositoryReconciler) sync(ctx context.Context, repository sourcev1. defer unlock() // archive artifact and check integrity - err = r.Storage.Archive(artifact, tmpGit, "", true) + err = r.Storage.Archive(artifact, tmpGit, true) if err != nil { err = fmt.Errorf("storage archive error: %w", err) return sourcev1.GitRepositoryNotReady(repository, sourcev1.StorageOperationFailedReason, err.Error()), err diff --git a/controllers/storage.go b/controllers/storage.go index a9098dae5..3eb699bbd 100644 --- a/controllers/storage.go +++ b/controllers/storage.go @@ -33,6 +33,12 @@ import ( "github.com/fluxcd/source-controller/internal/lockedfile" ) +const ( + excludeFile = ".sourceignore" + excludeVCS = ".git/,.gitignore,.gitmodules,.gitattributes" + defaultExcludes = "jpg,jpeg,gif,png,wmv,flv,tar.gz,zip" +) + // Storage manages artifacts type Storage struct { // BasePath is the local directory path where the source artifacts are stored. @@ -112,17 +118,22 @@ func (s *Storage) ArtifactExist(artifact sourcev1.Artifact) bool { return true } -// Archive creates a tar.gz to the artifact path from the given dir excluding the provided file extensions -func (s *Storage) Archive(artifact sourcev1.Artifact, dir string, excludes string, integrityCheck bool) error { - if excludes == "" { - excludes = "jpg,jpeg,gif,png,wmv,flv,tar.gz,zip" - } - +// Archive creates a tar.gz to the artifact path from the given dir excluding any VCS specific +// files and directories, or any of the excludes defined in the excludeFiles. +func (s *Storage) Archive(artifact sourcev1.Artifact, dir string, integrityCheck bool) error { ctx, cancel := context.WithTimeout(context.Background(), s.Timeout) defer cancel() - tarExcludes := fmt.Sprintf("--exclude=\\*.{%s} --exclude .git", excludes) - cmd := fmt.Sprintf("cd %s && tar -c %s -f - . | gzip > %s", dir, tarExcludes, artifact.Path) + var tarExcludes []string + if _, err := os.Stat(filepath.Join(dir, excludeFile)); !os.IsNotExist(err) { + tarExcludes = append(tarExcludes, "--exclude-file="+excludeFile) + } else { + tarExcludes = append(tarExcludes, fmt.Sprintf("--exclude=\\*.{%s}", defaultExcludes)) + } + for _, excl := range strings.Split(excludeVCS, ",") { + tarExcludes = append(tarExcludes, "--exclude="+excl) + } + cmd := fmt.Sprintf("cd %s && tar -c %s -f - . | gzip > %s", dir, strings.Join(tarExcludes, " "), artifact.Path) command := exec.CommandContext(ctx, "/bin/sh", "-c", cmd) err := command.Run()