Copy projects with symlinks

cmd/cnbBuild.go

@@ -39,6 +39,10 @@ type cnbBuildUtilsBundle struct {
 	*docker.Client
 }
 
+func (cnbBuildUtilsBundle) EvalSymlinks(path string) (string, error) {
+	return filepath.EvalSymlinks(path)
+}
+
 func processConfigs(main cnbBuildOptions, multipleImages []map[string]interface{}) ([]cnbBuildOptions, error) {
 	var result []cnbBuildOptions
 
@@ -493,7 +497,7 @@ func runCnbBuild(config *cnbBuildOptions, telemetry *buildpacks.Telemetry, image
 	}
 
 	if pathType != buildpacks.PathEnumArchive {
-		err = cnbutils.CopyProject(source, target, include, exclude, utils)
+		err = cnbutils.CopyProject(source, target, include, exclude, utils, true)
 		if err != nil {
 			log.SetErrorCategory(log.ErrorBuild)
 			return errors.Wrapf(err, "Copying  '%s' into '%s' failed", source, target)
@@ -619,7 +623,7 @@ func runCnbBuild(config *cnbBuildOptions, telemetry *buildpacks.Telemetry, image
 
 	if len(config.PreserveFiles) > 0 {
 		if pathType != buildpacks.PathEnumArchive {
-			err = cnbutils.CopyProject(target, source, ignore.CompileIgnoreLines(config.PreserveFiles...), nil, utils)
+			err = cnbutils.CopyProject(target, source, ignore.CompileIgnoreLines(config.PreserveFiles...), nil, utils, false)
 			if err != nil {
 				log.SetErrorCategory(log.ErrorBuild)
 				return errors.Wrapf(err, "failed to preserve files using glob '%s'", config.PreserveFiles)

cmd/gradleExecuteBuild.go

@@ -121,38 +121,22 @@ type Artifact struct {
 	Name string `json:"name,omitempty"`
 }
 
-type WalkDir func(root string, fn fs.WalkDirFunc) error
-
-type Filepath interface {
-	WalkDir(root string, fn fs.WalkDirFunc) error
-}
-
-type WalkDirFunc func(root string, fn fs.WalkDirFunc) error
-
-func (f WalkDirFunc) WalkDir(root string, fn fs.WalkDirFunc) error {
-	return f(root, fn)
-}
-
 type gradleExecuteBuildUtils interface {
 	command.ExecRunner
 	piperutils.FileUtils
-	Filepath
 }
 
 type gradleExecuteBuildUtilsBundle struct {
 	*command.Command
 	*piperutils.Files
-	Filepath
 }
 
 func newGradleExecuteBuildUtils() gradleExecuteBuildUtils {
-	var walkDirFunc WalkDirFunc = filepath.WalkDir
 	utils := gradleExecuteBuildUtilsBundle{
 		Command: &command.Command{
 			StepName: "gradleExecuteBuild",
 		},
-		Files:    &piperutils.Files{},
-		Filepath: walkDirFunc,
+		Files: &piperutils.Files{},
 	}
 	utils.Stdout(log.Writer())
 	utils.Stderr(log.Writer())

cmd/gradleExecuteBuild_test.go

@@ -21,7 +21,6 @@ const moduleFileContent = `{"variants": [{"name": "apiElements","files": [{"name
 type gradleExecuteBuildMockUtils struct {
 	*mock.ExecMockRunner
 	*mock.FilesMock
-	Filepath
 }
 
 type isDirEntryMock func() bool
@@ -121,16 +120,9 @@ func TestRunGradleExecuteBuild(t *testing.T) {
 	})
 
 	t.Run("success case - publishing of artifacts", func(t *testing.T) {
-		var walkDir WalkDirFunc = func(root string, fn fs.WalkDirFunc) error {
-			var dirMock isDirEntryMock = func() bool {
-				return false
-			}
-			return fn(filepath.Join("test_subproject_path", "build", "publications", "maven", "module.json"), dirMock, nil)
-		}
 		utils := gradleExecuteBuildMockUtils{
 			ExecMockRunner: &mock.ExecMockRunner{},
 			FilesMock:      &mock.FilesMock{},
-			Filepath:       walkDir,
 		}
 		utils.FilesMock.AddFile("path/to/build.gradle", []byte{})
 		utils.FilesMock.AddFile(filepath.Join("test_subproject_path", "build", "publications", "maven", "module.json"), []byte(moduleFileContent))

pkg/cnbutils/copy_project.go

@@ -1,72 +1,75 @@
 package cnbutils
 
 import (
+	"io/fs"
 	"os"
-	"path"
 	"path/filepath"
 	"strings"
 
 	"github.com/SAP/jenkins-library/pkg/log"
-	"github.com/pkg/errors"
 	ignore "github.com/sabhiram/go-gitignore"
 )
 
-func CopyProject(source, target string, include, exclude *ignore.GitIgnore, utils BuildUtils) error {
-	sourceFiles, _ := utils.Glob(path.Join(source, "**"))
-	for _, sourceFile := range sourceFiles {
-		relPath, err := filepath.Rel(source, sourceFile)
-		if err != nil {
-			log.SetErrorCategory(log.ErrorBuild)
-			return errors.Wrapf(err, "Calculating relative path for '%s' failed", sourceFile)
-		}
-		if !isIgnored(relPath, include, exclude) {
-			target := path.Join(target, strings.ReplaceAll(sourceFile, source, ""))
-			dir, err := utils.DirExists(sourceFile)
+func CopyProject(source, target string, include, exclude *ignore.GitIgnore, utils BuildUtils, follow bool) error {
+	log.Entry().Debugf("Copying project from '%s' to '%s' (following symlings %t)", source, target, follow)
+
+	utils.WalkDir(source, func(path string, d fs.DirEntry, err error) error {
+		target := filepath.Join(target, strings.ReplaceAll(path, source, ""))
+
+		switch d.Type() {
+		case fs.ModeDir:
+			log.Entry().Debugf("Creating directpry '%s'", path)
+			err := utils.MkdirAll(target, os.ModePerm)
+			if err != nil {
+				return err
+			}
+		case fs.ModeSymlink:
+			linkTarget, err := utils.EvalSymlinks(path)
+			if err != nil {
+				return err
+			}
+			stat, err := utils.Stat(linkTarget)
 			if err != nil {
-				log.SetErrorCategory(log.ErrorBuild)
-				return errors.Wrapf(err, "Checking file info '%s' failed", target)
+				return err
 			}
 
-			if dir {
-				err = utils.MkdirAll(target, os.ModePerm)
+			if follow && stat.IsDir() {
+				destination := target
+				if !filepath.IsAbs(linkTarget) {
+					destination, err = filepath.Abs(filepath.Join(target, linkTarget))
+					if err != nil {
+						return err
+					}
+				}
+				err = CopyProject(linkTarget, destination, include, exclude, utils, follow)
 				if err != nil {
-					log.SetErrorCategory(log.ErrorBuild)
-					return errors.Wrapf(err, "Creating directory '%s' failed", target)
+					return err
 				}
 			} else {
-				log.Entry().Debugf("Copying '%s' to '%s'", sourceFile, target)
-				err = copyFile(sourceFile, target, utils)
+				log.Entry().Debugf("Creating symlink from '%s' to '%s'", target, linkTarget)
+				err = utils.Symlink(linkTarget, target)
 				if err != nil {
-					log.SetErrorCategory(log.ErrorBuild)
-					return errors.Wrapf(err, "Copying '%s' to '%s' failed", sourceFile, target)
+					return err
+				}
+			}
+		default:
+			relPath, err := filepath.Rel(source, path)
+			if err != nil {
+				return err
+			}
+			if !isIgnored(relPath, include, exclude) {
+				log.Entry().Debugf("Copying file from '%s' to '%s'", path, target)
+				_, err = utils.Copy(path, target)
+				if err != nil {
+					return err
 				}
 			}
-
 		}
-	}
+		return nil
+	})
 	return nil
 }
 
-func copyFile(source, target string, utils BuildUtils) error {
-	targetDir := filepath.Dir(target)
-
-	exists, err := utils.DirExists(targetDir)
-	if err != nil {
-		return err
-	}
-
-	if !exists {
-		log.Entry().Debugf("Creating directory %s", targetDir)
-		err = utils.MkdirAll(targetDir, os.ModePerm)
-		if err != nil {
-			return err
-		}
-	}
-
-	_, err = utils.Copy(source, target)
-	return err
-}
-
 func isIgnored(find string, include, exclude *ignore.GitIgnore) bool {
 	if exclude != nil {
 		filtered := exclude.MatchesPath(find)

pkg/cnbutils/copy_project_test.go

@@ -13,27 +13,43 @@ import (
 )
 
 func TestCopyProject(t *testing.T) {
-	t.Run("copies file according to doublestart globs", func(t *testing.T) {
+	t.Run("copy project with following symlinks", func(t *testing.T) {
 		mockUtils := &cnbutils.MockUtils{
 			FilesMock: &mock.FilesMock{},
 		}
-		mockUtils.AddFile("workdir/src/test.yaml", []byte(""))
-		mockUtils.AddFile("workdir/src/subdir1/test2.yaml", []byte(""))
-		mockUtils.AddFile("workdir/src/subdir1/subdir2/test3.yaml", []byte(""))
-		err := cnbutils.CopyProject("workdir/src", "/dest", ignore.CompileIgnoreLines([]string{"**/*.yaml"}...), nil, mockUtils)
+		mockUtils.AddFile("/workdir/src/test.yaml", []byte(""))
+		mockUtils.AddFile("/workdir/src/subdir1/test2.yaml", []byte(""))
+		mockUtils.AddFile("/workdir/src/subdir1/subdir2/test3.yaml", []byte(""))
+
+		mockUtils.AddDir("/workdir/apps")
+		mockUtils.AddFile("/workdir/apps/foo.yaml", []byte(""))
+		mockUtils.Symlink("/workdir/apps", "/workdir/src/apps")
+
+		err := cnbutils.CopyProject("/workdir/src", "/dest", ignore.CompileIgnoreLines([]string{"**/*.yaml"}...), nil, mockUtils, true)
 		assert.NoError(t, err)
-		assert.True(t, mockUtils.HasCopiedFile("workdir/src/test.yaml", "/dest/test.yaml"))
-		assert.True(t, mockUtils.HasCopiedFile("workdir/src/subdir1/test2.yaml", "/dest/subdir1/test2.yaml"))
-		assert.True(t, mockUtils.HasCopiedFile("workdir/src/subdir1/subdir2/test3.yaml", "/dest/subdir1/subdir2/test3.yaml"))
+		assert.True(t, mockUtils.HasCopiedFile("/workdir/src/test.yaml", "/dest/test.yaml"))
+		assert.True(t, mockUtils.HasCopiedFile("/workdir/src/subdir1/test2.yaml", "/dest/subdir1/test2.yaml"))
+		assert.True(t, mockUtils.HasCopiedFile("/workdir/src/subdir1/subdir2/test3.yaml", "/dest/subdir1/subdir2/test3.yaml"))
+		assert.True(t, mockUtils.HasCopiedFile("/workdir/apps/foo.yaml", "/dest/apps/foo.yaml"))
 	})
 
-	t.Run("copies file according to simple globs", func(t *testing.T) {
+	t.Run("copy project without following symlinks", func(t *testing.T) {
 		mockUtils := &cnbutils.MockUtils{
 			FilesMock: &mock.FilesMock{},
 		}
-		mockUtils.AddFile("src/test.yaml", []byte(""))
-		err := cnbutils.CopyProject("src", "/dest", ignore.CompileIgnoreLines([]string{"*.yaml"}...), nil, mockUtils)
+		mockUtils.AddFile("/workdir/src/test.yaml", []byte(""))
+		mockUtils.AddFile("/workdir/src/subdir1/test2.yaml", []byte(""))
+		mockUtils.AddFile("/workdir/src/subdir1/subdir2/test3.yaml", []byte(""))
+
+		mockUtils.AddDir("/workdir/apps")
+		mockUtils.AddFile("/workdir/apps/foo.yaml", []byte(""))
+		mockUtils.Symlink("/workdir/apps", "/workdir/src/apps")
+
+		err := cnbutils.CopyProject("/workdir/src", "/dest", ignore.CompileIgnoreLines([]string{"**/*.yaml"}...), nil, mockUtils, false)
 		assert.NoError(t, err)
-		assert.True(t, mockUtils.HasCopiedFile("src/test.yaml", "/dest/test.yaml"))
+		assert.True(t, mockUtils.HasCopiedFile("/workdir/src/test.yaml", "/dest/test.yaml"))
+		assert.True(t, mockUtils.HasCopiedFile("/workdir/src/subdir1/test2.yaml", "/dest/subdir1/test2.yaml"))
+		assert.True(t, mockUtils.HasCopiedFile("/workdir/src/subdir1/subdir2/test3.yaml", "/dest/subdir1/subdir2/test3.yaml"))
+		assert.True(t, mockUtils.HasCreatedSymlink("/workdir/apps", "/workdir/src/apps"))
 	})
 }

pkg/cnbutils/utils.go

@@ -10,4 +10,5 @@ type BuildUtils interface {
 	command.ExecRunner
 	piperutils.FileUtils
 	docker.Download
+	EvalSymlinks(path string) (string, error)
 }

pkg/mock/fileUtils.go

@@ -11,6 +11,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"io/fs"
 	"os"
 	"path/filepath"
 	"sort"
@@ -540,8 +541,18 @@ func (f *FilesMock) Symlink(oldname, newname string) error {
 	f.init()
 
 	f.files[newname] = &fileProperties{
-		isLink: true,
-		target: oldname,
+		isLink:  true,
+		target:  oldname,
+		content: &[]byte{},
+	}
+
+	isDir, err := f.DirExists(oldname)
+	if err != nil {
+		return err
+	}
+
+	if isDir {
+		f.files[newname].content = &dirContent
 	}
 
 	return nil
@@ -700,3 +711,61 @@ func (f *FilesMockRelativeGlob) Glob(pattern string) ([]string, error) {
 	sort.Strings(matches)
 	return matches, nil
 }
+
+type dirEntry struct {
+	name  string
+	isDir bool
+	mode  fs.FileMode
+}
+
+func (d dirEntry) Name() string {
+	return d.name
+}
+
+func (d dirEntry) IsDir() bool {
+	return d.isDir
+}
+
+func (d dirEntry) Type() fs.FileMode {
+	return d.mode
+}
+
+func (d dirEntry) Info() (fs.FileInfo, error) {
+	panic("not implemented")
+}
+
+func (f *FilesMock) Readlink(name string) (string, error) {
+	properties, ok := f.files[name]
+	if ok && properties.isLink {
+		return properties.target, nil
+	}
+	return "", fmt.Errorf("could not retrieve target for %s", name)
+}
+
+func (f *FilesMock) EvalSymlinks(path string) (string, error) {
+	return f.Readlink(path)
+}
+
+func (f *FilesMock) WalkDir(root string, fn fs.WalkDirFunc) error {
+	for name, properties := range f.files {
+		if root != "." && !strings.HasPrefix(name, root) {
+			continue
+		}
+		dirEntry := dirEntry{
+			name: filepath.Base(name),
+			mode: fs.ModePerm,
+		}
+
+		if properties.isDir() {
+			dirEntry.mode = fs.ModeDir
+			dirEntry.isDir = true
+		}
+
+		if properties.isLink {
+			dirEntry.mode = fs.ModeSymlink
+		}
+
+		fn(name, dirEntry, nil)
+	}
+	return nil
+}

pkg/piperutils/fileUtils.go

@@ -43,6 +43,9 @@ type FileUtils interface {
 	CurrentTime(format string) string
 	Open(name string) (io.ReadWriteCloser, error)
 	Create(name string) (io.ReadWriteCloser, error)
+	WalkDir(root string, fn fs.WalkDirFunc) error
+	Readlink(name string) (string, error)
+	Stat(path string) (os.FileInfo, error)
 }
 
 // Files ...
@@ -513,3 +516,13 @@ func (f Files) Open(name string) (io.ReadWriteCloser, error) {
 func (f Files) Create(name string) (io.ReadWriteCloser, error) {
 	return os.Create(name)
 }
+
+// WalkDir wraps filepath.WalkDir
+func (f Files) WalkDir(root string, fn fs.WalkDirFunc) error {
+	return filepath.WalkDir(root, fn)
+}
+
+// Readlink wraps os.Readlink
+func (f Files) Readlink(name string) (string, error) {
+	return os.Readlink(name)
+}