new dirWalk for test purposes (#11277)

closes #10086 but it uses old lib (https://github.com/karrick/godirwalk). This branch could be used for tests for someone who experiences troubles with RAM with RemoveContents func. (For example for this guy https://discord.com/channels/687972960811745322/1233600171821240380) Maybe we should fork this lib :) thing for future milestone --------- Co-authored-by: JkLondon <[email protected]> Co-authored-by: alex.sharov <[email protected]>
erigontech · Jul 24, 2024 · 842b5f7 · 842b5f7
1 parent 62ed361
commit 842b5f7
Show file tree

Hide file tree

Showing 3 changed files with 27 additions and 19 deletions.
diff --git a/eth/backend.go b/eth/backend.go
@@ -24,7 +24,6 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"io/fs"
 	"math"
 	"math/big"
 	"net"
@@ -37,6 +36,8 @@ import (
 	"sync/atomic"
 	"time"
 
+	"github.com/karrick/godirwalk"
+
 	"github.com/erigontech/mdbx-go/mdbx"
 	lru "github.com/hashicorp/golang-lru/arc/v2"
 	"github.com/holiman/uint256"
@@ -52,7 +53,6 @@ import (
 	libcommon "github.com/erigontech/erigon-lib/common"
 	"github.com/erigontech/erigon-lib/common/datadir"
 	"github.com/erigontech/erigon-lib/common/dbg"
-	"github.com/erigontech/erigon-lib/common/dir"
 	"github.com/erigontech/erigon-lib/common/disk"
 	"github.com/erigontech/erigon-lib/common/mem"
 	"github.com/erigontech/erigon-lib/config3"
@@ -1679,26 +1679,31 @@ func (s *Ethereum) ExecutionModule() *eth1.EthereumExecutionModule {
 
 // RemoveContents is like os.RemoveAll, but preserve dir itself
 func RemoveContents(dirname string) error {
-	d, err := os.Open(dirname)
-	if err != nil {
-		if errors.Is(err, fs.ErrNotExist) {
-			// ignore due to windows
-			_ = os.MkdirAll(dirname, 0o755)
-			return nil
-		}
-		return err
-	}
-	defer d.Close()
-	files, err := dir.ReadDir(dirname)
+	err := godirwalk.Walk(dirname, &godirwalk.Options{
+		ErrorCallback: func(s string, err error) godirwalk.ErrorAction {
+			if os.IsNotExist(err) {
+				return godirwalk.SkipNode
+			}
+			return godirwalk.Halt
+		},
+		FollowSymbolicLinks: true,
+		Unsorted:            true,
+		Callback: func(osPathname string, d *godirwalk.Dirent) error {
+			if osPathname == dirname {
+				return nil
+			}
+			if d.IsSymlink() {
+				return nil
+			}
+			return os.RemoveAll(filepath.Join(dirname, d.Name()))
+		},
+		PostChildrenCallback: nil,
+		ScratchBuffer:        nil,
+		AllowNonDirectory:    false,
+	})
 	if err != nil {
 		return err
 	}
-	for _, file := range files {
-		err = os.RemoveAll(filepath.Join(dirname, file.Name()))
-		if err != nil {
-			return err
-		}
-	}
 	return nil
 }
 

diff --git a/go.mod b/go.mod
@@ -61,6 +61,7 @@ require (
 	github.com/jedib0t/go-pretty/v6 v6.5.9
 	github.com/json-iterator/go v1.1.12
 	github.com/julienschmidt/httprouter v1.3.0
+	github.com/karrick/godirwalk v1.17.0
 	github.com/klauspost/compress v1.17.8
 	github.com/libp2p/go-libp2p v0.34.0
 	github.com/libp2p/go-libp2p-mplex v0.9.0

diff --git a/go.sum b/go.sum
@@ -516,6 +516,8 @@ github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfV
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U=
 github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
+github.com/karrick/godirwalk v1.17.0 h1:b4kY7nqDdioR/6qnbHQyDvmA17u5G1cZ6J+CZXwSWoI=
+github.com/karrick/godirwalk v1.17.0/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk=
 github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=