From 60d404a2908799f84d618732afaad2efd2ba1f01 Mon Sep 17 00:00:00 2001 From: Predrag Rogic Date: Wed, 17 Jan 2024 00:48:12 +0000 Subject: [PATCH 1/2] workaround for "runc list" returning "no such file or directory" --- pkg/minikube/cruntime/cri.go | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/pkg/minikube/cruntime/cri.go b/pkg/minikube/cruntime/cri.go index 1244defb0ee5..8460b482c246 100644 --- a/pkg/minikube/cruntime/cri.go +++ b/pkg/minikube/cruntime/cri.go @@ -25,11 +25,13 @@ import ( "os/exec" "path" "strings" + "time" "github.com/blang/semver/v4" "github.com/pkg/errors" "k8s.io/klog/v2" "k8s.io/minikube/pkg/minikube/command" + "k8s.io/minikube/pkg/util/retry" ) // container maps to 'runc list -f json' @@ -108,10 +110,23 @@ func listCRIContainers(cr CommandRunner, root string, o ListContainersOptions) ( } args = append(args, "list", "-f", "json") - rr, err = cr.RunCmd(exec.Command("sudo", args...)) - if err != nil { + + // retry only on "no such file or directory" error returned by runc + // TODO (prezha): remove this workaround when #17976 is addressed upstream and we've updated to that runc version + list := func() error { + rr, err = cr.RunCmd(exec.Command("sudo", args...)) + if errors.Is(err, os.ErrNotExist) { + klog.Infof("temporary error listing containers using runc (will retry): %v", err) + return err + } + // don't retry on non-error or any error other than os.ErrNotExist, but retain the underlying error and check thereafter + return nil + } + // return any underlying error + if rerr := retry.Expo(list, time.Millisecond*100, time.Second); rerr != nil || err != nil { return nil, errors.Wrap(err, "runc") } + content := rr.Stdout.Bytes() klog.Infof("JSON = %s", content) d := json.NewDecoder(bytes.NewReader(content)) From c2f1b814d7e0ecd388589338f61424aa14cb7195 Mon Sep 17 00:00:00 2001 From: Predrag Rogic Date: Wed, 17 Jan 2024 21:12:14 +0000 Subject: [PATCH 2/2] retry on "runc list" error --- pkg/minikube/cruntime/cri.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/pkg/minikube/cruntime/cri.go b/pkg/minikube/cruntime/cri.go index 8460b482c246..4993cd0fb362 100644 --- a/pkg/minikube/cruntime/cri.go +++ b/pkg/minikube/cruntime/cri.go @@ -111,19 +111,18 @@ func listCRIContainers(cr CommandRunner, root string, o ListContainersOptions) ( args = append(args, "list", "-f", "json") - // retry only on "no such file or directory" error returned by runc - // TODO (prezha): remove this workaround when #17976 is addressed upstream and we've updated to that runc version + // avoid "no such file or directory" runc list error by retrying + // TODO (prezha): consider removing this retry workaround when #17976 is addressed upstream and we've updated to that runc version list := func() error { rr, err = cr.RunCmd(exec.Command("sudo", args...)) - if errors.Is(err, os.ErrNotExist) { + if err != nil { klog.Infof("temporary error listing containers using runc (will retry): %v", err) return err } - // don't retry on non-error or any error other than os.ErrNotExist, but retain the underlying error and check thereafter + // bail out return nil } - // return any underlying error - if rerr := retry.Expo(list, time.Millisecond*100, time.Second); rerr != nil || err != nil { + if err := retry.Expo(list, 100*time.Millisecond, 5*time.Second); err != nil { return nil, errors.Wrap(err, "runc") }