From 34f1057bfab1a0bec3d19e4b4cd0fcdfcb9d442a Mon Sep 17 00:00:00 2001 From: Johannes Dillmann Date: Fri, 6 Sep 2024 14:27:17 +0200 Subject: [PATCH] Copy full project (#5033) --- cmd/cnbBuild.go | 4 +- pkg/cnbutils/copy_project.go | 81 ++++++++++++++++++++++++++++--- pkg/cnbutils/copy_project_test.go | 28 ++++++++--- pkg/mock/fileUtils.go | 17 ++++++- pkg/piperutils/fileUtils.go | 13 +++++ 5 files changed, 125 insertions(+), 18 deletions(-) diff --git a/cmd/cnbBuild.go b/cmd/cnbBuild.go index 61dc9803a6..1bc012df79 100644 --- a/cmd/cnbBuild.go +++ b/cmd/cnbBuild.go @@ -493,7 +493,7 @@ func runCnbBuild(config *cnbBuildOptions, telemetry *buildpacks.Telemetry, image } if pathType != buildpacks.PathEnumArchive { - err = cnbutils.CopyProject(source, target, include, exclude, utils) + err = cnbutils.CopyProject(source, target, include, exclude, utils, false) if err != nil { log.SetErrorCategory(log.ErrorBuild) return errors.Wrapf(err, "Copying '%s' into '%s' failed", source, target) @@ -619,7 +619,7 @@ func runCnbBuild(config *cnbBuildOptions, telemetry *buildpacks.Telemetry, image if len(config.PreserveFiles) > 0 { if pathType != buildpacks.PathEnumArchive { - err = cnbutils.CopyProject(target, source, ignore.CompileIgnoreLines(config.PreserveFiles...), nil, utils) + err = cnbutils.CopyProject(target, source, ignore.CompileIgnoreLines(config.PreserveFiles...), nil, utils, true) if err != nil { log.SetErrorCategory(log.ErrorBuild) return errors.Wrapf(err, "failed to preserve files using glob '%s'", config.PreserveFiles) diff --git a/pkg/cnbutils/copy_project.go b/pkg/cnbutils/copy_project.go index 1d20892afd..d7f423820f 100644 --- a/pkg/cnbutils/copy_project.go +++ b/pkg/cnbutils/copy_project.go @@ -1,6 +1,7 @@ package cnbutils import ( + "io/fs" "os" "path" "path/filepath" @@ -11,23 +12,82 @@ import ( ignore "github.com/sabhiram/go-gitignore" ) -func CopyProject(source, target string, include, exclude *ignore.GitIgnore, utils BuildUtils) error { - sourceFiles, _ := utils.Glob(path.Join(source, "**")) +func shouldBeFiltered(path string, knownSymlinks []string) bool { + for _, symlink := range knownSymlinks { + if strings.HasPrefix(path, symlink) { + return true + } + } + return false +} + +func filterSymlinks(sourceFiles []string, utils BuildUtils) ([]string, error) { + filteredFiles := []string{} + knownSymlinks := []string{} + + for _, sourceFile := range sourceFiles { + if shouldBeFiltered(sourceFile, knownSymlinks) { + continue + } + + isSymlink, err := symlinkExists(sourceFile, utils) + if err != nil { + return nil, err + } + + if isSymlink { + log.Entry().Debugf("Ignoring any path below %q", sourceFile) + knownSymlinks = append(knownSymlinks, sourceFile) + } + filteredFiles = append(filteredFiles, sourceFile) + } + return filteredFiles, nil +} + +func CopyProject(source, target string, include, exclude *ignore.GitIgnore, utils BuildUtils, follow bool) error { + sourceFiles, err := utils.Glob(path.Join(source, "**")) + if err != nil { + return err + } + + if !follow { + sourceFiles, err = filterSymlinks(sourceFiles, utils) + if err != nil { + return err + } + } + for _, sourceFile := range sourceFiles { relPath, err := filepath.Rel(source, sourceFile) if err != nil { log.SetErrorCategory(log.ErrorBuild) return errors.Wrapf(err, "Calculating relative path for '%s' failed", sourceFile) } + if !isIgnored(relPath, include, exclude) { target := path.Join(target, strings.ReplaceAll(sourceFile, source, "")) - dir, err := utils.DirExists(sourceFile) + + isSymlink, err := symlinkExists(sourceFile, utils) if err != nil { - log.SetErrorCategory(log.ErrorBuild) - return errors.Wrapf(err, "Checking file info '%s' failed", target) + return err } - if dir { + isDir, err := utils.DirExists(sourceFile) + if err != nil { + return err + } + + if isSymlink { + linkTarget, err := utils.Readlink(sourceFile) + if err != nil { + return err + } + log.Entry().Debugf("Creating symlink from %q to %q", target, linkTarget) + err = utils.Symlink(linkTarget, target) + if err != nil { + return err + } + } else if isDir { err = utils.MkdirAll(target, os.ModePerm) if err != nil { log.SetErrorCategory(log.ErrorBuild) @@ -41,12 +101,16 @@ func CopyProject(source, target string, include, exclude *ignore.GitIgnore, util return errors.Wrapf(err, "Copying '%s' to '%s' failed", sourceFile, target) } } - } } return nil } +func symlinkExists(path string, utils BuildUtils) (bool, error) { + lstat, err := utils.Lstat(path) + return lstat.Mode().Type() == fs.ModeSymlink, err +} + func copyFile(source, target string, utils BuildUtils) error { targetDir := filepath.Dir(target) @@ -56,13 +120,14 @@ func copyFile(source, target string, utils BuildUtils) error { } if !exists { - log.Entry().Debugf("Creating directory %s", targetDir) + log.Entry().Debugf("Creating directory '%s'", targetDir) err = utils.MkdirAll(targetDir, os.ModePerm) if err != nil { return err } } + log.Entry().Debugf("Copying '%s' to '%s'", source, target) _, err = utils.Copy(source, target) return err } diff --git a/pkg/cnbutils/copy_project_test.go b/pkg/cnbutils/copy_project_test.go index abd706fa4a..fcdd4e818b 100644 --- a/pkg/cnbutils/copy_project_test.go +++ b/pkg/cnbutils/copy_project_test.go @@ -13,27 +13,43 @@ import ( ) func TestCopyProject(t *testing.T) { - t.Run("copies file according to doublestart globs", func(t *testing.T) { + t.Run("copy project with following symlinks", func(t *testing.T) { mockUtils := &cnbutils.MockUtils{ FilesMock: &mock.FilesMock{}, } mockUtils.AddFile("workdir/src/test.yaml", []byte("")) mockUtils.AddFile("workdir/src/subdir1/test2.yaml", []byte("")) mockUtils.AddFile("workdir/src/subdir1/subdir2/test3.yaml", []byte("")) - err := cnbutils.CopyProject("workdir/src", "/dest", ignore.CompileIgnoreLines([]string{"**/*.yaml"}...), nil, mockUtils) + + mockUtils.AddDir("workdir/apps") + mockUtils.AddFile("workdir/apps/foo.yaml", []byte("")) + mockUtils.Symlink("workdir/apps", "/workdir/src/apps") + + err := cnbutils.CopyProject("workdir/src", "/dest", ignore.CompileIgnoreLines([]string{"**"}...), nil, mockUtils, true) assert.NoError(t, err) assert.True(t, mockUtils.HasCopiedFile("workdir/src/test.yaml", "/dest/test.yaml")) assert.True(t, mockUtils.HasCopiedFile("workdir/src/subdir1/test2.yaml", "/dest/subdir1/test2.yaml")) assert.True(t, mockUtils.HasCopiedFile("workdir/src/subdir1/subdir2/test3.yaml", "/dest/subdir1/subdir2/test3.yaml")) + assert.True(t, mockUtils.HasCopiedFile("workdir/src/apps", "/dest/apps")) }) - t.Run("copies file according to simple globs", func(t *testing.T) { + t.Run("copy project without following symlinks", func(t *testing.T) { mockUtils := &cnbutils.MockUtils{ FilesMock: &mock.FilesMock{}, } - mockUtils.AddFile("src/test.yaml", []byte("")) - err := cnbutils.CopyProject("src", "/dest", ignore.CompileIgnoreLines([]string{"*.yaml"}...), nil, mockUtils) + mockUtils.AddFile("workdir/src/test.yaml", []byte("")) + mockUtils.AddFile("workdir/src/subdir1/test2.yaml", []byte("")) + mockUtils.AddFile("workdir/src/subdir1/subdir2/test3.yaml", []byte("")) + + mockUtils.AddDir("workdir/apps") + mockUtils.AddFile("workdir/apps/foo.yaml", []byte("")) + mockUtils.Symlink("workdir/apps", "/workdir/src/apps") + + err := cnbutils.CopyProject("workdir/src", "/dest", ignore.CompileIgnoreLines([]string{"**/*.yaml"}...), nil, mockUtils, false) assert.NoError(t, err) - assert.True(t, mockUtils.HasCopiedFile("src/test.yaml", "/dest/test.yaml")) + assert.True(t, mockUtils.HasCopiedFile("workdir/src/test.yaml", "/dest/test.yaml")) + assert.True(t, mockUtils.HasCopiedFile("workdir/src/subdir1/test2.yaml", "/dest/subdir1/test2.yaml")) + assert.True(t, mockUtils.HasCopiedFile("workdir/src/subdir1/subdir2/test3.yaml", "/dest/subdir1/subdir2/test3.yaml")) + assert.True(t, mockUtils.HasCreatedSymlink("workdir/apps", "/workdir/src/apps")) }) } diff --git a/pkg/mock/fileUtils.go b/pkg/mock/fileUtils.go index ab36a5d001..b8d13ff6cd 100644 --- a/pkg/mock/fileUtils.go +++ b/pkg/mock/fileUtils.go @@ -486,6 +486,10 @@ func (f *FilesMock) Stat(path string) (os.FileInfo, error) { }, nil } +func (f *FilesMock) Lstat(path string) (os.FileInfo, error) { + return f.Stat(path) +} + // Chmod changes the file mode for the entry at the given path func (f *FilesMock) Chmod(path string, mode os.FileMode) error { props, exists := f.files[f.toAbsPath(path)] @@ -540,8 +544,9 @@ func (f *FilesMock) Symlink(oldname, newname string) error { f.init() f.files[newname] = &fileProperties{ - isLink: true, - target: oldname, + isLink: true, + target: oldname, + content: &[]byte{}, } return nil @@ -700,3 +705,11 @@ func (f *FilesMockRelativeGlob) Glob(pattern string) ([]string, error) { sort.Strings(matches) return matches, nil } + +func (f *FilesMock) Readlink(name string) (string, error) { + properties, ok := f.files[name] + if ok && properties.isLink { + return properties.target, nil + } + return "", fmt.Errorf("could not retrieve target for %s", name) +} diff --git a/pkg/piperutils/fileUtils.go b/pkg/piperutils/fileUtils.go index 4cc5091ba6..ce54dc741b 100644 --- a/pkg/piperutils/fileUtils.go +++ b/pkg/piperutils/fileUtils.go @@ -43,6 +43,9 @@ type FileUtils interface { CurrentTime(format string) string Open(name string) (io.ReadWriteCloser, error) Create(name string) (io.ReadWriteCloser, error) + Readlink(name string) (string, error) + Stat(path string) (os.FileInfo, error) + Lstat(path string) (os.FileInfo, error) } // Files ... @@ -513,3 +516,13 @@ func (f Files) Open(name string) (io.ReadWriteCloser, error) { func (f Files) Create(name string) (io.ReadWriteCloser, error) { return os.Create(name) } + +// Readlink wraps os.Readlink +func (f Files) Readlink(name string) (string, error) { + return os.Readlink(name) +} + +// Readlink wraps os.Readlink +func (f Files) Lstat(path string) (os.FileInfo, error) { + return os.Lstat(path) +}