From 1fea3edf8ced43868a1759c0d9fc8d9f67a2a1a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Wed, 15 Feb 2023 13:14:41 +0100 Subject: [PATCH] fix: always absolute paths when tarring files to a Docker container (#814) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore: extract to variable to avoid double calculation * chore: add tests including absolute paths * chore: merge tests using a test table * fix: keep parent directory as the root of the TAR file * docs: document the change * chore: make the path OS-agnostic * chore: simplify * fix: use filepath to be OS-independent * chore(deps): bump github.com/jackc/pgx/v4 in /examples/cockroachdb (#819) Bumps [github.com/jackc/pgx/v4](https://github.com/jackc/pgx) from 4.17.2 to 4.18.0. - [Release notes](https://github.com/jackc/pgx/releases) - [Changelog](https://github.com/jackc/pgx/blob/master/CHANGELOG.md) - [Commits](https://github.com/jackc/pgx/compare/v4.17.2...v4.18.0) --- updated-dependencies: - dependency-name: github.com/jackc/pgx/v4 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * chore(deps): bump golang.org/x/sys from 0.4.0 to 0.5.0 (#816) Bumps [golang.org/x/sys](https://github.com/golang/sys) from 0.4.0 to 0.5.0. - [Release notes](https://github.com/golang/sys/releases) - [Commits](https://github.com/golang/sys/compare/v0.4.0...v0.5.0) --- updated-dependencies: - dependency-name: golang.org/x/sys dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * chore(deps): bump golang.org/x/text from 0.6.0 to 0.7.0 (#818) Bumps [golang.org/x/text](https://github.com/golang/text) from 0.6.0 to 0.7.0. - [Release notes](https://github.com/golang/text/releases) - [Commits](https://github.com/golang/text/compare/v0.6.0...v0.7.0) --- updated-dependencies: - dependency-name: golang.org/x/text dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * chore(deps): bump github.com/containerd/containerd from 1.6.16 to 1.6.17 (#817) Bumps [github.com/containerd/containerd](https://github.com/containerd/containerd) from 1.6.16 to 1.6.17. - [Release notes](https://github.com/containerd/containerd/releases) - [Changelog](https://github.com/containerd/containerd/blob/main/RELEASES.md) - [Commits](https://github.com/containerd/containerd/compare/v1.6.16...v1.6.17) --- updated-dependencies: - dependency-name: github.com/containerd/containerd dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * chore(deps): bump google.golang.org/grpc from 1.52.3 to 1.53.0 in /examples (#827) * chore(deps): bump google.golang.org/grpc in /examples/pubsub Bumps [google.golang.org/grpc](https://github.com/grpc/grpc-go) from 1.52.3 to 1.53.0. - [Release notes](https://github.com/grpc/grpc-go/releases) - [Commits](https://github.com/grpc/grpc-go/compare/v1.52.3...v1.53.0) --- updated-dependencies: - dependency-name: google.golang.org/grpc dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * chore(deps): bump google.golang.org/grpc in /examples/spanner Bumps [google.golang.org/grpc](https://github.com/grpc/grpc-go) from 1.52.3 to 1.53.0. - [Release notes](https://github.com/grpc/grpc-go/releases) - [Commits](https://github.com/grpc/grpc-go/compare/v1.52.3...v1.53.0) --- updated-dependencies: - dependency-name: google.golang.org/grpc dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * chore(deps): bump google.golang.org/grpc in /examples/bigtable Bumps [google.golang.org/grpc](https://github.com/grpc/grpc-go) from 1.52.3 to 1.53.0. - [Release notes](https://github.com/grpc/grpc-go/releases) - [Commits](https://github.com/grpc/grpc-go/compare/v1.52.3...v1.53.0) --- updated-dependencies: - dependency-name: google.golang.org/grpc dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * chore(deps): bump google.golang.org/grpc in /examples/datastore Bumps [google.golang.org/grpc](https://github.com/grpc/grpc-go) from 1.52.3 to 1.53.0. - [Release notes](https://github.com/grpc/grpc-go/releases) - [Commits](https://github.com/grpc/grpc-go/compare/v1.52.3...v1.53.0) --- updated-dependencies: - dependency-name: google.golang.org/grpc dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * chore(deps): bump google.golang.org/grpc in /examples/firestore Bumps [google.golang.org/grpc](https://github.com/grpc/grpc-go) from 1.52.3 to 1.53.0. - [Release notes](https://github.com/grpc/grpc-go/releases) - [Commits](https://github.com/grpc/grpc-go/compare/v1.52.3...v1.53.0) --- updated-dependencies: - dependency-name: google.golang.org/grpc dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * feat: add localstack module (#800) * chore: add localstack basic example Run "go run . --name localstack --image "localstack/localstack:0.11.2" --title LocalStack" * chore: add wait for log * chore: add Docker socket binding * chore: support defining legacy mode * chore: support passing services and configuring them * chore: simplify the creation of a localstack container request It will allow users not to pass the legacy mode everywhere. Instead, it's defined in the initial request, and consumed downstream * chore: support configuring the AWS region * chore: expose localstack container * chore: support for retrieving the endpoint of a given service * chore: store enabled services in the localstack container * chore: store the region in the localstack container * chore: apply default region when needed * chore: support passing version and legacyMode as functional options * chore: support for overriding the container request with a functional option * chore: remove useless method for retrieving the internal endpoint of a service * chore: simplify unit tests * chore: create aws session using v1 * chore: add S3 tests * chore: rename start localstack function * chore: define default functions as vars * chore: add tests for legacy mode * chore: move the Session code to the tests We want to et the user define how to get the Session * docs: use S3 in the docs * fix: read the expected daemon host from the provider * chore: move v1 tests to its own test package * feat: add example test for S3 using v2 * chore: store localstack credentials in a struct * feat: create a functional opt for passing the AWS credentials * chore: define defaults for the AWS credentials * chore: include a test for overriding the image * chore: remove withCredentials Set them up using the override, as it's as simple as adding the right env vars * chore: remove withRegion Set it up using the override, as it's as simple as adding the right env var * chore: remove withVersion Set them up using the override, as it's as simple as setting the req.Image * chore: rename configure method * chore: reorder container configuration 1. merge the override request 2. set up the functional opts 3. configure Docker host * chore: move to types * chor: export structs * docs: document exported functions * docs: document how to create localstack container * docs: reorder * docs: document containerRequest override * docs: document the container functional opts * chore: be more explicit in comment Co-authored-by: Eddú Meléndez Gonzales * chore: convert into real module * chore: remove support for legacy mode * chore: move tests * fix: update module paths * fix: update tests after moving an example to modules * chore: bump default version to 1.3.1 * chore: remove Port from the Service struct If the user wants to use the legacy mode, then they should build the environment accordingly (e.g. SERVICES env var) * fix: wrong relative path to the docs * chore: remove any code to Services We'll delegate it to the user, using the localstack env vars * chore: adjust unit tests * docs: include tests * chore: use waitFor HTTP health * chore: bump to latest version * chore: go mod examples * chore: rename GH action workflow file --------- Co-authored-by: Eddú Meléndez Gonzales * chore(deps): bump go.mongodb.org/mongo-driver in /examples/mongodb (#831) Bumps [go.mongodb.org/mongo-driver](https://github.com/mongodb/mongo-go-driver) from 1.11.1 to 1.11.2. - [Release notes](https://github.com/mongodb/mongo-go-driver/releases) - [Commits](https://github.com/mongodb/mongo-go-driver/compare/v1.11.1...v1.11.2) --- updated-dependencies: - dependency-name: go.mongodb.org/mongo-driver dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * chore(deps): bump google.golang.org/api from 0.109.0 to 0.110.0 in /examples (#836) * chore(deps): bump google.golang.org/api in /examples/bigtable Bumps [google.golang.org/api](https://github.com/googleapis/google-api-go-client) from 0.109.0 to 0.110.0. - [Release notes](https://github.com/googleapis/google-api-go-client/releases) - [Changelog](https://github.com/googleapis/google-api-go-client/blob/main/CHANGES.md) - [Commits](https://github.com/googleapis/google-api-go-client/compare/v0.109.0...v0.110.0) --- updated-dependencies: - dependency-name: google.golang.org/api dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * chore(deps): bump google.golang.org/api in /examples/spanner Bumps [google.golang.org/api](https://github.com/googleapis/google-api-go-client) from 0.109.0 to 0.110.0. - [Release notes](https://github.com/googleapis/google-api-go-client/releases) - [Changelog](https://github.com/googleapis/google-api-go-client/blob/main/CHANGES.md) - [Commits](https://github.com/googleapis/google-api-go-client/compare/v0.109.0...v0.110.0) --- updated-dependencies: - dependency-name: google.golang.org/api dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * chore(deps): bump google.golang.org/api in /examples/pubsub Bumps [google.golang.org/api](https://github.com/googleapis/google-api-go-client) from 0.109.0 to 0.110.0. - [Release notes](https://github.com/googleapis/google-api-go-client/releases) - [Changelog](https://github.com/googleapis/google-api-go-client/blob/main/CHANGES.md) - [Commits](https://github.com/googleapis/google-api-go-client/compare/v0.109.0...v0.110.0) --- updated-dependencies: - dependency-name: google.golang.org/api dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * chore(deps): bump google.golang.org/api in /examples/firestore Bumps [google.golang.org/api](https://github.com/googleapis/google-api-go-client) from 0.109.0 to 0.110.0. - [Release notes](https://github.com/googleapis/google-api-go-client/releases) - [Changelog](https://github.com/googleapis/google-api-go-client/blob/main/CHANGES.md) - [Commits](https://github.com/googleapis/google-api-go-client/compare/v0.109.0...v0.110.0) --- updated-dependencies: - dependency-name: google.golang.org/api dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * chore(deps): bump google.golang.org/api in /examples/datastore Bumps [google.golang.org/api](https://github.com/googleapis/google-api-go-client) from 0.109.0 to 0.110.0. - [Release notes](https://github.com/googleapis/google-api-go-client/releases) - [Changelog](https://github.com/googleapis/google-api-go-client/blob/main/CHANGES.md) - [Commits](https://github.com/googleapis/google-api-go-client/compare/v0.109.0...v0.110.0) --- updated-dependencies: - dependency-name: google.golang.org/api dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * feat: support generating Go modules or example modules (#826) * chore: create a separate module for the generator * chore: get the root directory in the new layout * feat: support for creating modules or examples * fix: update dependabot and mkdocs * chore: extract example parent dir calculation to a function * chore: define entrypoint and container name methods * chore: rename setupTech to startContainer * chore: rename setup function in example modules * chor: include modulegen in dependabot * chore: add build script for modulegen * chore: include modulegen tests in CI * fix: update tests * docs: add missing dependabot entry * chore: update GH workflow example type * fix: do not depend on hardcoded lengths * chore: generate modules in its own directory in docs * chore: add build script for modules * docs: reuse * fix: always pass the absolute path to the tar code --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Eddú Meléndez Gonzales --- container.go | 8 ++++++ docker_test.go | 27 ++++++++++++++---- file.go | 21 +++++++++++--- file_test.go | 77 ++++++++++++++++++++++++++++++++------------------ 4 files changed, 96 insertions(+), 37 deletions(-) diff --git a/container.go b/container.go index c23dfef228..70146ab9a2 100644 --- a/container.go +++ b/container.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "io" + "path/filepath" "time" "github.com/docker/docker/api/types" @@ -234,6 +235,13 @@ func (c *ContainerRequest) GetContext() (io.Reader, error) { return c.ContextArchive, nil } + // always pass context as absolute path + abs, err := filepath.Abs(c.Context) + if err != nil { + return nil, fmt.Errorf("error getting absolute path: %w", err) + } + c.Context = abs + buildContext, err := archive.TarWithOptions(c.Context, &archive.TarOptions{}) if err != nil { return nil, err diff --git a/docker_test.go b/docker_test.go index 08894b9b2e..faa934d031 100644 --- a/docker_test.go +++ b/docker_test.go @@ -2022,18 +2022,20 @@ func TestDockerContainerCopyDirToContainer(t *testing.T) { Started: true, }) + p := filepath.Join(".", "testresources", "Dokerfile") require.NoError(t, err) terminateContainerOnEnd(t, ctx, nginxC) - err = nginxC.CopyDirToContainer(ctx, "./testresources/Dockerfile", "/tmp/testresources/Dockerfile", 700) + err = nginxC.CopyDirToContainer(ctx, p, "/tmp/testresources/Dockerfile", 700) require.Error(t, err) // copying a file using the directory method will raise an error - err = nginxC.CopyDirToContainer(ctx, "./testresources", "/tmp/testresources", 700) + p = filepath.Join(".", "testresources") + err = nginxC.CopyDirToContainer(ctx, p, "/tmp/testresources", 700) if err != nil { t.Fatal(err) } - assertExtractedFiles(t, ctx, nginxC, "./testresources", "/tmp/testresources/") + assertExtractedFiles(t, ctx, nginxC, p, "/tmp/testresources/") } func TestDockerCreateContainerWithFiles(t *testing.T) { @@ -2109,15 +2111,27 @@ func TestDockerCreateContainerWithDirs(t *testing.T) { ctx := context.Background() hostDirName := "testresources" + abs, err := filepath.Abs(filepath.Join(".", hostDirName)) + assert.Nil(t, err) + tests := []struct { name string dir ContainerFile hasError bool }{ + { + name: "success copy directory with full path", + dir: ContainerFile{ + HostFilePath: abs, + ContainerFilePath: "/tmp/" + hostDirName, // the parent dir must exist + FileMode: 700, + }, + hasError: false, + }, { name: "success copy directory", dir: ContainerFile{ - HostFilePath: "./" + hostDirName, + HostFilePath: filepath.Join("./", hostDirName), ContainerFilePath: "/tmp/" + hostDirName, // the parent dir must exist FileMode: 700, }, @@ -2561,10 +2575,11 @@ func assertExtractedFiles(t *testing.T, ctx context.Context, container Container require.NoError(t, err) } + fp := filepath.Join(containerFilePath, srcFile.Name()) // copy file by file, as there is a limitation in the Docker client to copy an entiry directory from the container // paths for the container files are using Linux path separators - fd, err := container.CopyFileFromContainer(ctx, containerFilePath+"/"+srcFile.Name()) - require.NoError(t, err, "Path not found in container: %s", containerFilePath+"/"+srcFile.Name()) + fd, err := container.CopyFileFromContainer(ctx, fp) + require.NoError(t, err, "Path not found in container: %s", fp) defer fd.Close() targetPath := filepath.Join(tmpDir, srcFile.Name()) diff --git a/file.go b/file.go index 1f402f3842..509da643d4 100644 --- a/file.go +++ b/file.go @@ -8,6 +8,7 @@ import ( "io" "os" "path/filepath" + "strings" ) func isDir(path string) (bool, error) { @@ -31,6 +32,13 @@ func isDir(path string) (bool, error) { // tarDir compress a directory using tar + gzip algorithms func tarDir(src string, fileMode int64) (*bytes.Buffer, error) { + // always pass src as absolute path + abs, err := filepath.Abs(src) + if err != nil { + return &bytes.Buffer{}, fmt.Errorf("error getting absolute path: %w", err) + } + src = abs + buffer := &bytes.Buffer{} fmt.Printf(">> creating TAR file from directory: %s\n", src) @@ -39,8 +47,12 @@ func tarDir(src string, fileMode int64) (*bytes.Buffer, error) { zr := gzip.NewWriter(buffer) tw := tar.NewWriter(zr) + _, baseDir := filepath.Split(src) + // keep the path relative to the parent directory + index := strings.LastIndex(src, baseDir) + // walk through every file in the folder - err := filepath.Walk(src, func(file string, fi os.FileInfo, errFn error) error { + err = filepath.Walk(src, func(file string, fi os.FileInfo, errFn error) error { if errFn != nil { return fmt.Errorf("error traversing the file system: %w", errFn) } @@ -57,9 +69,10 @@ func tarDir(src string, fileMode int64) (*bytes.Buffer, error) { return fmt.Errorf("error getting file info header: %w", err) } - // must provide real name - // (see https://golang.org/src/archive/tar/common.go?#L626) - header.Name = filepath.ToSlash(file) + // see https://pkg.go.dev/archive/tar#FileInfoHeader: + // Since fs.FileInfo's Name method only returns the base name of the file it describes, + // it may be necessary to modify Header.Name to provide the full path name of the file. + header.Name = filepath.ToSlash(file[index:]) header.Mode = fileMode // write header diff --git a/file_test.go b/file_test.go index c34acf7060..74a020378b 100644 --- a/file_test.go +++ b/file_test.go @@ -12,6 +12,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func Test_IsDir(t *testing.T) { @@ -53,38 +54,60 @@ func Test_IsDir(t *testing.T) { } func Test_TarDir(t *testing.T) { - src := filepath.Join(".", "testresources") - - buff, err := tarDir(src, 0755) - if err != nil { - t.Fatal(err) + originalSrc := filepath.Join(".", "testresources") + tests := []struct { + abs bool + }{ + { + abs: false, + }, + { + abs: true, + }, } - tmpDir := filepath.Join(t.TempDir(), "subfolder") - err = untar(tmpDir, bytes.NewReader(buff.Bytes())) - if err != nil { - t.Fatal(err) - } + for _, test := range tests { + t.Run(fmt.Sprintf("TarDir with abs=%t", test.abs), func(t *testing.T) { + src := originalSrc + if test.abs { + absSrc, err := filepath.Abs(src) + require.Nil(t, err) - srcFiles, err := os.ReadDir(src) - if err != nil { - log.Fatal(err) - } + src = absSrc + } - for _, srcFile := range srcFiles { - if srcFile.IsDir() { - continue - } - srcBytes, err := os.ReadFile(filepath.Join(src, srcFile.Name())) - if err != nil { - t.Fatal(err) - } + buff, err := tarDir(src, 0755) + if err != nil { + t.Fatal(err) + } - untarBytes, err := os.ReadFile(filepath.Join(tmpDir, "testresources", srcFile.Name())) - if err != nil { - t.Fatal(err) - } - assert.Equal(t, srcBytes, untarBytes) + tmpDir := filepath.Join(t.TempDir(), "subfolder") + err = untar(tmpDir, bytes.NewReader(buff.Bytes())) + if err != nil { + t.Fatal(err) + } + + srcFiles, err := os.ReadDir(src) + if err != nil { + log.Fatal(err) + } + + for _, srcFile := range srcFiles { + if srcFile.IsDir() { + continue + } + srcBytes, err := os.ReadFile(filepath.Join(src, srcFile.Name())) + if err != nil { + t.Fatal(err) + } + + untarBytes, err := os.ReadFile(filepath.Join(tmpDir, "testresources", srcFile.Name())) + if err != nil { + t.Fatal(err) + } + assert.Equal(t, srcBytes, untarBytes) + } + }) } }