From df3d52d54b3ef5c60bab95e36016cfcdffbf6d1e Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Thu, 6 Apr 2023 09:05:26 +0200 Subject: [PATCH] Best effort copying of `run` directory (#2448) Best effort copying of `run` directory (#2448) --- NOTICE.txt | 10 +-- go.mod | 3 +- go.sum | 8 +- .../application/upgrade/test/case1/README.md | 0 .../application/upgrade/test/case2/README.md | 1 + .../application/upgrade/test/case3/README.md | 1 + .../pkg/agent/application/upgrade/upgrade.go | 21 +++-- .../agent/application/upgrade/upgrade_test.go | 80 +++++++++++++++++++ 8 files changed, 109 insertions(+), 15 deletions(-) create mode 100644 internal/pkg/agent/application/upgrade/test/case1/README.md create mode 100644 internal/pkg/agent/application/upgrade/test/case2/README.md create mode 100644 internal/pkg/agent/application/upgrade/test/case3/README.md diff --git a/NOTICE.txt b/NOTICE.txt index f04730e95d1..333eb38ee90 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -3902,12 +3902,12 @@ Contents of probable licence file $GOMODCACHE/github.com/oklog/ulid@v1.3.1/LICEN -------------------------------------------------------------------------------- -Dependency : github.com/otiai10/copy -Version: v1.2.0 +Dependency : github.com/michalpristas/copy +Version: v1.9.1 Licence type (autodetected): MIT -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/otiai10/copy@v1.2.0/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/michalpristas/copy@v1.9.1/LICENSE: The MIT License (MIT) @@ -13466,11 +13466,11 @@ Contents of probable licence file $GOMODCACHE/github.com/opencontainers/image-sp -------------------------------------------------------------------------------- Dependency : github.com/otiai10/mint -Version: v1.3.1 +Version: v1.4.1 Licence type (autodetected): MIT -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/otiai10/mint@v1.3.1/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/otiai10/mint@v1.4.1/LICENSE: Copyright 2017 otiai10 (Hiromu OCHIAI) diff --git a/go.mod b/go.mod index a42e2293f11..3712440b507 100644 --- a/go.mod +++ b/go.mod @@ -33,7 +33,7 @@ require ( github.com/mitchellh/hashstructure v1.1.0 github.com/mitchellh/mapstructure v1.5.0 github.com/oklog/ulid v1.3.1 - github.com/otiai10/copy v1.2.0 + github.com/otiai10/copy v1.9.0 github.com/pierrre/gotestcover v0.0.0-20160517101806-924dca7d15f0 github.com/pkg/errors v0.9.1 github.com/rs/zerolog v1.27.0 @@ -156,6 +156,7 @@ replace ( github.com/dop251/goja => github.com/andrewkroh/goja v0.0.0-20190128172624-dd2ac4456e20 github.com/dop251/goja_nodejs => github.com/dop251/goja_nodejs v0.0.0-20171011081505-adff31b136e6 github.com/fsnotify/fsnotify => github.com/adriansr/fsnotify v1.4.8-0.20211018144411-a81f2b630e7c + github.com/otiai10/copy v1.9.0 => github.com/michalpristas/copy v1.9.1 github.com/tonistiigi/fifo => github.com/containerd/fifo v0.0.0-20190816180239-bda0ff6ed73c ) diff --git a/go.sum b/go.sum index 9dbad58f874..044f500d969 100644 --- a/go.sum +++ b/go.sum @@ -894,6 +894,8 @@ github.com/maxbrunsfeld/counterfeiter/v6 v6.2.2/go.mod h1:eD9eIE7cdwcMi9rYluz88J github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/mholt/archiver/v3 v3.5.1/go.mod h1:e3dqJ7H78uzsRSEACH1joayhuSyhnonssnDhppzS1L4= +github.com/michalpristas/copy v1.9.1 h1:jYTonyfnuxrOjb/LKwho7U++S6/iBuW1MyeTNzG7wyc= +github.com/michalpristas/copy v1.9.1/go.mod h1:DiseS1dNi8N2JrJVHtzng5BgHfIMMP6Y4DOP1hqJCvM= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= @@ -1012,13 +1014,11 @@ github.com/opencontainers/selinux v1.6.0/go.mod h1:VVGKuOLlE7v4PJyT6h7mNWvq1rzqi github.com/opencontainers/selinux v1.8.0/go.mod h1:RScLhm78qiWa2gbVCcGkC7tCGdgk3ogry1nUQF8Evvo= github.com/opencontainers/selinux v1.8.2/go.mod h1:MUIHuUEvKB1wtJjQdOyYRgOnLD2xAPP8dBsCoU0KuF8= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= -github.com/otiai10/copy v1.2.0 h1:HvG945u96iNadPoG2/Ja2+AUJeW5YuFQMixq9yirC+k= -github.com/otiai10/copy v1.2.0/go.mod h1:rrF5dJ5F0t/EWSYODDu4j9/vEeYHMkc8jt0zJChqQWw= github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE= github.com/otiai10/curr v1.0.0/go.mod h1:LskTG5wDwr8Rs+nNQ+1LlxRjAtTZZjtJW4rMXl6j4vs= github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo= -github.com/otiai10/mint v1.3.1 h1:BCmzIS3n71sGfHB5NMNDB3lHYPz8fWSkCAErHed//qc= -github.com/otiai10/mint v1.3.1/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH1OTc= +github.com/otiai10/mint v1.4.1 h1:HOVBfKP1oXIc0wWo9hZ8JLdZtyCPWqjvmFDuVZ0yv2Y= +github.com/otiai10/mint v1.4.1/go.mod h1:gifjb2MYOoULtKLqUAEILUG/9KONW6f7YsJ6vQLTlFI= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-buffruneio v0.2.0/go.mod h1:JkE26KsDizTr40EUHkXVtNPvgGtbSNq5BcowyYOWdKo= diff --git a/internal/pkg/agent/application/upgrade/test/case1/README.md b/internal/pkg/agent/application/upgrade/test/case1/README.md new file mode 100644 index 00000000000..e69de29bb2d diff --git a/internal/pkg/agent/application/upgrade/test/case2/README.md b/internal/pkg/agent/application/upgrade/test/case2/README.md new file mode 100644 index 00000000000..30d74d25844 --- /dev/null +++ b/internal/pkg/agent/application/upgrade/test/case2/README.md @@ -0,0 +1 @@ +test \ No newline at end of file diff --git a/internal/pkg/agent/application/upgrade/test/case3/README.md b/internal/pkg/agent/application/upgrade/test/case3/README.md new file mode 100644 index 00000000000..81403e41bd3 --- /dev/null +++ b/internal/pkg/agent/application/upgrade/test/case3/README.md @@ -0,0 +1 @@ +test 2 \ No newline at end of file diff --git a/internal/pkg/agent/application/upgrade/upgrade.go b/internal/pkg/agent/application/upgrade/upgrade.go index 8108fdb24ac..765985ea5fc 100644 --- a/internal/pkg/agent/application/upgrade/upgrade.go +++ b/internal/pkg/agent/application/upgrade/upgrade.go @@ -263,7 +263,7 @@ func copyRunDirectory(log *logger.Logger, newHash string) error { return errors.New(err, "failed to create run directory") } - err := copyDir(oldRunPath, newRunPath) + err := copyDir(log, oldRunPath, newRunPath, true) if os.IsNotExist(err) { // nothing to copy, operation ok log.Debugw("Run directory not present", "old_run_path", oldRunPath) @@ -279,7 +279,7 @@ func copyRunDirectory(log *logger.Logger, newHash string) error { // shutdownCallback returns a callback function to be executing during shutdown once all processes are closed. // this goes through runtime directory of agent and copies all the state files created by processes to new versioned // home directory with updated process name to match new version. -func shutdownCallback(_ *logger.Logger, homePath, prevVersion, newVersion, newHash string) reexec.ShutdownCallbackFn { +func shutdownCallback(l *logger.Logger, homePath, prevVersion, newVersion, newHash string) reexec.ShutdownCallbackFn { if release.Snapshot() { // SNAPSHOT is part of newVersion prevVersion += "-SNAPSHOT" @@ -297,7 +297,7 @@ func shutdownCallback(_ *logger.Logger, homePath, prevVersion, newVersion, newHa for _, processDir := range processDirs { newDir := strings.ReplaceAll(processDir, prevVersion, newVersion) newDir = strings.ReplaceAll(newDir, oldHome, newHome) - if err := copyDir(processDir, newDir); err != nil { + if err := copyDir(l, processDir, newDir, true); err != nil { return err } } @@ -343,11 +343,22 @@ func readDirs(dir string) ([]string, error) { return dirs, nil } -func copyDir(from, to string) error { +func copyDir(l *logger.Logger, from, to string, ignoreErrs bool) error { + var onErr func(error) error + + if ignoreErrs { + onErr = func(err error) error { + // ignore all errors, just log them + l.Infof("ignoring error: failed to copy %q to %q: %s", from, to, err.Error()) + return nil + } + } + return copy.Copy(from, to, copy.Options{ OnSymlink: func(_ string) copy.SymlinkAction { return copy.Shallow }, - Sync: true, + Sync: true, + OnErr: onErr, }) } diff --git a/internal/pkg/agent/application/upgrade/upgrade_test.go b/internal/pkg/agent/application/upgrade/upgrade_test.go index 7f0934296d3..84a2977e84b 100644 --- a/internal/pkg/agent/application/upgrade/upgrade_test.go +++ b/internal/pkg/agent/application/upgrade/upgrade_test.go @@ -9,15 +9,95 @@ import ( "io/ioutil" "os" "path/filepath" + "runtime" "strings" "testing" + "github.com/gofrs/flock" "github.com/stretchr/testify/require" "github.com/elastic/elastic-agent/internal/pkg/release" "github.com/elastic/elastic-agent/pkg/core/logger" ) +func Test_CopyFile(t *testing.T) { + l, _ := logger.New("test", false) + tt := []struct { + Name string + From string + To string + IgnoreErr bool + KeepOpen bool + ExpectedErr bool + }{ + { + "Existing, no onerr", + filepath.Join(".", "test", "case1", "README.md"), + filepath.Join(".", "test", "case1", "copy", "README.md"), + false, + false, + false, + }, + { + "Existing but open", + filepath.Join(".", "test", "case2", "README.md"), + filepath.Join(".", "test", "case2", "copy", "README.md"), + false, + true, + runtime.GOOS == "windows", // this fails only on, + }, + { + "Existing but open, ignore errors", + filepath.Join(".", "test", "case3", "README.md"), + filepath.Join(".", "test", "case3", "copy", "README.md"), + true, + true, + false, + }, + { + "Not existing, accept errors", + filepath.Join(".", "test", "case4", "README.md"), + filepath.Join(".", "test", "case4", "copy", "README.md"), + false, + false, + true, + }, + { + "Not existing, ignore errors", + filepath.Join(".", "test", "case4", "README.md"), + filepath.Join(".", "test", "case4", "copy", "README.md"), + true, + false, + false, + }, + } + + for _, tc := range tt { + t.Run(tc.Name, func(t *testing.T) { + defer func() { + // cleanup + _ = os.RemoveAll(filepath.Dir(tc.To)) + }() + + var fl *flock.Flock + if tc.KeepOpen { + // this uses syscalls to create inter-process lock + fl = flock.New(tc.From) + _, err := fl.TryLock() + require.NoError(t, err) + + defer func() { + require.NoError(t, fl.Unlock()) + }() + + } + + err := copyDir(l, tc.From, tc.To, tc.IgnoreErr) + require.Equal(t, tc.ExpectedErr, err != nil) + }) + } +} + func TestShutdownCallback(t *testing.T) { l, _ := logger.New("test", false) tmpDir, err := ioutil.TempDir("", "shutdown-test-")