From 5a082406ed019c3cdfac97afc436f3645a56f8cb Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Sat, 20 Apr 2024 10:20:23 +0700 Subject: [PATCH 01/14] fs: to be less smart - it's up to APP create file or not --- erigon-lib/downloader/torrent_files.go | 21 +++++---------------- erigon-lib/downloader/webseed.go | 2 +- 2 files changed, 6 insertions(+), 17 deletions(-) diff --git a/erigon-lib/downloader/torrent_files.go b/erigon-lib/downloader/torrent_files.go index ddaf62f7da1..7ef0c81e841 100644 --- a/erigon-lib/downloader/torrent_files.go +++ b/erigon-lib/downloader/torrent_files.go @@ -50,26 +50,22 @@ func (tf *TorrentFiles) delete(name string) error { return os.Remove(filepath.Join(tf.dir, name)) } -func (tf *TorrentFiles) Create(name string, res []byte) (ts *torrent.TorrentSpec, prohibited, created bool, err error) { +func (tf *TorrentFiles) Create(name string, res []byte) (ts *torrent.TorrentSpec, created bool, err error) { tf.lock.Lock() defer tf.lock.Unlock() - prohibited, err = tf.newDownloadsAreProhibited(name) - if err != nil { - return nil, false, false, err - } - if !tf.exists(name) && !prohibited { + if !tf.exists(name) { err = tf.create(name, res) if err != nil { - return nil, false, false, err + return nil, false, err } } ts, err = tf.load(filepath.Join(tf.dir, name)) if err != nil { - return nil, false, false, err + return nil, false, err } - return ts, prohibited, false, nil + return ts, false, nil } func (tf *TorrentFiles) create(name string, res []byte) error { @@ -136,13 +132,6 @@ func (tf *TorrentFiles) CreateWithMetaInfo(info *metainfo.Info, additionalMetaIn tf.lock.Lock() defer tf.lock.Unlock() - prohibited, err := tf.newDownloadsAreProhibited(name) - if err != nil { - return false, err - } - if prohibited { - return false, nil - } if tf.exists(name) { return false, nil } diff --git a/erigon-lib/downloader/webseed.go b/erigon-lib/downloader/webseed.go index 65a01215400..34a41e022bd 100644 --- a/erigon-lib/downloader/webseed.go +++ b/erigon-lib/downloader/webseed.go @@ -598,7 +598,7 @@ func (d *WebSeeds) DownloadAndSaveTorrentFile(ctx context.Context, name string) d.logger.Log(d.verbosity, "[snapshots] .torrent from webseed rejected", "name", name, "err", err) continue // it's ok if some HTTP provider failed - try next one } - ts, _, _, err = d.torrentFiles.Create(name, res) + ts, _, err = d.torrentFiles.Create(name, res) return ts, ts != nil, err } From 23e6e6f1bebf07f9f0d2bf2ee9efbae717a7f310 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Sat, 20 Apr 2024 10:37:31 +0700 Subject: [PATCH 02/14] save --- erigon-lib/downloader/downloader.go | 1 - 1 file changed, 1 deletion(-) diff --git a/erigon-lib/downloader/downloader.go b/erigon-lib/downloader/downloader.go index 471fc0d6f6f..ba9dbb33b67 100644 --- a/erigon-lib/downloader/downloader.go +++ b/erigon-lib/downloader/downloader.go @@ -2327,7 +2327,6 @@ func (d *Downloader) AddMagnetLink(ctx context.Context, infoHash metainfo.Hash, } t, ok, err := addTorrentFile(ctx, spec, d.torrentClient, d.db, d.webseeds) - if err != nil { return err } From 845bc3c30247f6cd03573e44b6eda1a106029039 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Sat, 20 Apr 2024 11:37:43 +0700 Subject: [PATCH 03/14] save --- erigon-lib/common/dir/rw_dir.go | 11 +++--- erigon-lib/downloader/torrent_files.go | 48 +++++++++++++------------- 2 files changed, 30 insertions(+), 29 deletions(-) diff --git a/erigon-lib/common/dir/rw_dir.go b/erigon-lib/common/dir/rw_dir.go index 769ba0f3cd2..a4e4d69bc31 100644 --- a/erigon-lib/common/dir/rw_dir.go +++ b/erigon-lib/common/dir/rw_dir.go @@ -72,15 +72,16 @@ func WriteFileWithFsync(name string, data []byte, perm os.FileMode) error { return err } defer f.Close() - _, err = f.Write(data) - if err != nil { + if _, err = f.Write(data); err != nil { return err } - err = f.Sync() - if err != nil { + if err = f.Sync(); err != nil { + return err + } + if err = f.Close(); err != nil { return err } - return err + return nil } func Recreate(dir string) { diff --git a/erigon-lib/downloader/torrent_files.go b/erigon-lib/downloader/torrent_files.go index 7ef0c81e841..c3599e3e8d5 100644 --- a/erigon-lib/downloader/torrent_files.go +++ b/erigon-lib/downloader/torrent_files.go @@ -177,43 +177,34 @@ func (tf *TorrentFiles) ProhibitNewDownloads(t string) error { } func (tf *TorrentFiles) prohibitNewDownloads(t string) error { - // open or create file ProhibitNewDownloadsFileName - f, err := os.OpenFile(filepath.Join(tf.dir, ProhibitNewDownloadsFileName), os.O_CREATE|os.O_RDONLY, 0644) - if err != nil { - return fmt.Errorf("open file: %w", err) - } - defer f.Close() + fPath := filepath.Join(tf.dir, ProhibitNewDownloadsFileName) + exist := dir.FileExist(fPath) + var prohibitedList []string - torrentListJsonBytes, err := io.ReadAll(f) - if err != nil { - return fmt.Errorf("read file: %w", err) - } - if len(torrentListJsonBytes) > 0 { - if err := json.Unmarshal(torrentListJsonBytes, &prohibitedList); err != nil { - return fmt.Errorf("unmarshal: %w", err) + if exist { + torrentListJsonBytes, err := os.ReadFile(fPath) + if err != nil { + return fmt.Errorf("read file: %w", err) + } + if len(torrentListJsonBytes) > 0 { + if err := json.Unmarshal(torrentListJsonBytes, &prohibitedList); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } } } if slices.Contains(prohibitedList, t) { return nil } prohibitedList = append(prohibitedList, t) - f.Close() - // write new prohibited list by opening the file in truncate mode - f, err = os.OpenFile(filepath.Join(tf.dir, ProhibitNewDownloadsFileName), os.O_TRUNC|os.O_WRONLY, 0644) - if err != nil { - return fmt.Errorf("open file for writing: %w", err) - } - defer f.Close() prohibitedListJsonBytes, err := json.Marshal(prohibitedList) if err != nil { return fmt.Errorf("marshal: %w", err) } - if _, err := f.Write(prohibitedListJsonBytes); err != nil { + if err := dir.WriteFileWithFsync(fPath, prohibitedListJsonBytes, 0644); err != nil { return fmt.Errorf("write: %w", err) } - - return f.Sync() + return nil } func (tf *TorrentFiles) NewDownloadsAreProhibited(name string) (bool, error) { @@ -223,7 +214,13 @@ func (tf *TorrentFiles) NewDownloadsAreProhibited(name string) (bool, error) { } func (tf *TorrentFiles) newDownloadsAreProhibited(name string) (bool, error) { - f, err := os.OpenFile(filepath.Join(tf.dir, ProhibitNewDownloadsFileName), os.O_CREATE|os.O_APPEND|os.O_RDONLY, 0644) + fPath := filepath.Join(tf.dir, ProhibitNewDownloadsFileName) + exists := dir.FileExist(fPath) + if !exists { + return false, nil + } + + f, err := os.OpenFile(fPath, os.O_RDONLY, 0644) if err != nil { return false, err } @@ -233,6 +230,9 @@ func (tf *TorrentFiles) newDownloadsAreProhibited(name string) (bool, error) { if err != nil { return false, fmt.Errorf("NewDownloadsAreProhibited: read file: %w", err) } + if exists && len(torrentListJsonBytes) == 0 { // backward compatibility: if .lock exists and empty - it means everything is prohibited + return true, nil + } if len(torrentListJsonBytes) > 0 { if err := json.Unmarshal(torrentListJsonBytes, &prohibitedList); err != nil { return false, fmt.Errorf("NewDownloadsAreProhibited: unmarshal: %w", err) From 59e68d6a25b5da58947145b2318fa58ce55835fe Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Sat, 20 Apr 2024 11:45:38 +0700 Subject: [PATCH 04/14] save --- erigon-lib/downloader/torrent_files_test.go | 50 +++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 erigon-lib/downloader/torrent_files_test.go diff --git a/erigon-lib/downloader/torrent_files_test.go b/erigon-lib/downloader/torrent_files_test.go new file mode 100644 index 00000000000..0766d3b24f6 --- /dev/null +++ b/erigon-lib/downloader/torrent_files_test.go @@ -0,0 +1,50 @@ +package downloader + +import ( + "os" + "path/filepath" + "testing" + + "github.com/ledgerwatch/erigon-lib/common/datadir" + "github.com/stretchr/testify/require" +) + +func TestFSProhibitBackwardCompat(t *testing.T) { + require := require.New(t) + dirs := datadir.New(t.TempDir()) + + //prev version of .lock - is empty .lock file which exitence prohibiting everything + t.Run("no prev version .lock", func(t *testing.T) { + tf := NewAtomicTorrentFiles(dirs.Snap) + prohibited, err := tf.NewDownloadsAreProhibited("v1-004900-005000-headers.seg") + require.NoError(err) + require.False(prohibited) + prohibited, err = tf.NewDownloadsAreProhibited("v1-004900-005000-headers.seg.torrent") + require.NoError(err) + require.False(prohibited) + }) + t.Run("prev version .lock support", func(t *testing.T) { + err := os.WriteFile(filepath.Join(dirs.Snap, ProhibitNewDownloadsFileName), nil, 0644) + require.NoError(err) + + tf := NewAtomicTorrentFiles(dirs.Snap) + prohibited, err := tf.NewDownloadsAreProhibited("v1-004900-005000-headers.seg") + require.NoError(err) + require.True(prohibited) + prohibited, err = tf.NewDownloadsAreProhibited("v1-004900-005000-headers.seg.torrent") + require.NoError(err) + require.True(prohibited) + }) + t.Run("prev version .lock upgrade", func(t *testing.T) { + err := os.WriteFile(filepath.Join(dirs.Snap, ProhibitNewDownloadsFileName), nil, 0644) + require.NoError(err) + + tf := NewAtomicTorrentFiles(dirs.Snap) + prohibited, err := tf.NewDownloadsAreProhibited("v1-004900-005000-headers.seg") + require.NoError(err) + require.True(prohibited) + prohibited, err = tf.NewDownloadsAreProhibited("v1-004900-005000-headers.seg.torrent") + require.NoError(err) + require.True(prohibited) + }) +} From bdbd776c6198b562994ef20caabfc0527c10b251 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Sat, 20 Apr 2024 14:22:09 +0700 Subject: [PATCH 05/14] save --- erigon-lib/downloader/torrent_files_test.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/erigon-lib/downloader/torrent_files_test.go b/erigon-lib/downloader/torrent_files_test.go index 0766d3b24f6..3cbbd3204cb 100644 --- a/erigon-lib/downloader/torrent_files_test.go +++ b/erigon-lib/downloader/torrent_files_test.go @@ -36,15 +36,19 @@ func TestFSProhibitBackwardCompat(t *testing.T) { require.True(prohibited) }) t.Run("prev version .lock upgrade", func(t *testing.T) { + //old lock err := os.WriteFile(filepath.Join(dirs.Snap, ProhibitNewDownloadsFileName), nil, 0644) require.NoError(err) tf := NewAtomicTorrentFiles(dirs.Snap) + err = tf.prohibitNewDownloads("transactions") //upgrade + require.NoError(err) + prohibited, err := tf.NewDownloadsAreProhibited("v1-004900-005000-headers.seg") require.NoError(err) - require.True(prohibited) + require.False(prohibited) prohibited, err = tf.NewDownloadsAreProhibited("v1-004900-005000-headers.seg.torrent") require.NoError(err) - require.True(prohibited) + require.False(prohibited) }) } From 6ef03d8bce5f70cab3732a79f562236b4474eaa5 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Sun, 21 Apr 2024 20:44:50 +0700 Subject: [PATCH 06/14] merge devel --- erigon-lib/downloader/torrent_files_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/erigon-lib/downloader/torrent_files_test.go b/erigon-lib/downloader/torrent_files_test.go index 3cbbd3204cb..a936f1f3970 100644 --- a/erigon-lib/downloader/torrent_files_test.go +++ b/erigon-lib/downloader/torrent_files_test.go @@ -15,7 +15,7 @@ func TestFSProhibitBackwardCompat(t *testing.T) { //prev version of .lock - is empty .lock file which exitence prohibiting everything t.Run("no prev version .lock", func(t *testing.T) { - tf := NewAtomicTorrentFiles(dirs.Snap) + tf := NewAtomicTorrentFS(dirs.Snap) prohibited, err := tf.NewDownloadsAreProhibited("v1-004900-005000-headers.seg") require.NoError(err) require.False(prohibited) @@ -27,7 +27,7 @@ func TestFSProhibitBackwardCompat(t *testing.T) { err := os.WriteFile(filepath.Join(dirs.Snap, ProhibitNewDownloadsFileName), nil, 0644) require.NoError(err) - tf := NewAtomicTorrentFiles(dirs.Snap) + tf := NewAtomicTorrentFS(dirs.Snap) prohibited, err := tf.NewDownloadsAreProhibited("v1-004900-005000-headers.seg") require.NoError(err) require.True(prohibited) @@ -40,7 +40,7 @@ func TestFSProhibitBackwardCompat(t *testing.T) { err := os.WriteFile(filepath.Join(dirs.Snap, ProhibitNewDownloadsFileName), nil, 0644) require.NoError(err) - tf := NewAtomicTorrentFiles(dirs.Snap) + tf := NewAtomicTorrentFS(dirs.Snap) err = tf.prohibitNewDownloads("transactions") //upgrade require.NoError(err) From 5625836d44f12f705ef9674083b52866576ec6fc Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Thu, 25 Apr 2024 10:35:46 +0700 Subject: [PATCH 07/14] save --- erigon-lib/go.mod | 2 +- erigon-lib/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/erigon-lib/go.mod b/erigon-lib/go.mod index 7b66a4a6892..3f13cfb766c 100644 --- a/erigon-lib/go.mod +++ b/erigon-lib/go.mod @@ -5,7 +5,7 @@ go 1.20 require ( github.com/erigontech/mdbx-go v0.27.24 github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20240417163500-185a51876901 - github.com/ledgerwatch/interfaces v0.0.0-20240320062914-b57f05746087 + github.com/ledgerwatch/interfaces v0.0.0-20240425033437-5355fd37741e github.com/ledgerwatch/log/v3 v3.9.0 github.com/ledgerwatch/secp256k1 v1.0.0 ) diff --git a/erigon-lib/go.sum b/erigon-lib/go.sum index 5ea19433598..53b0f12fd56 100644 --- a/erigon-lib/go.sum +++ b/erigon-lib/go.sum @@ -258,8 +258,8 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20240417163500-185a51876901 h1:gAcI47OHnt/1e/APIV0093NVdviIfAnBUzFyybmKL1Q= github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20240417163500-185a51876901/go.mod h1:3AuPxZc85jkehh/HA9h8gabv5MSi3kb/ddtzBsTVJFo= -github.com/ledgerwatch/interfaces v0.0.0-20240320062914-b57f05746087 h1:Y59HUAT/+02Qbm6g7MuY7i8E0kUihPe7+ftDnR8oQzQ= -github.com/ledgerwatch/interfaces v0.0.0-20240320062914-b57f05746087/go.mod h1:ugQv1QllJzBny3cKZKxUrSnykkjkBgm27eQM6dnGAcc= +github.com/ledgerwatch/interfaces v0.0.0-20240425033437-5355fd37741e h1:NsjLpSpKotdzk61ne5pxCHSMJP+cRQEHZVRJp5FOsXI= +github.com/ledgerwatch/interfaces v0.0.0-20240425033437-5355fd37741e/go.mod h1:ugQv1QllJzBny3cKZKxUrSnykkjkBgm27eQM6dnGAcc= github.com/ledgerwatch/log/v3 v3.9.0 h1:iDwrXe0PVwBC68Dd94YSsHbMgQ3ufsgjzXtFNFVZFRk= github.com/ledgerwatch/log/v3 v3.9.0/go.mod h1:EiAY6upmI/6LkNhOVxb4eVsmsP11HZCnZ3PlJMjYiqE= github.com/ledgerwatch/secp256k1 v1.0.0 h1:Usvz87YoTG0uePIV8woOof5cQnLXGYa162rFf3YnwaQ= From aff444a6d11099e2e33979f9d89643a5c7389ec4 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Thu, 25 Apr 2024 11:16:07 +0700 Subject: [PATCH 08/14] .lock to store whiteList instead of blackList --- erigon-lib/direct/downloader_client.go | 4 +- erigon-lib/direct/sentry_client_mock.go | 16 +- .../downloader/downloader_grpc_server.go | 5 +- erigon-lib/downloader/torrent_files.go | 75 +++--- erigon-lib/downloader/torrent_files_test.go | 3 +- erigon-lib/go.mod | 2 +- erigon-lib/go.sum | 4 +- .../gointerfaces/downloader/downloader.pb.go | 241 ++++++++++++------ .../downloader/downloader_grpc.pb.go | 50 ++-- turbo/snapshotsync/snapshotsync.go | 34 ++- 10 files changed, 259 insertions(+), 175 deletions(-) diff --git a/erigon-lib/direct/downloader_client.go b/erigon-lib/direct/downloader_client.go index 319e3bcd1d2..63ba7cb477b 100644 --- a/erigon-lib/direct/downloader_client.go +++ b/erigon-lib/direct/downloader_client.go @@ -36,8 +36,8 @@ func (c *DownloaderClient) Add(ctx context.Context, in *proto_downloader.AddRequ return c.server.Add(ctx, in) } -func (c *DownloaderClient) ProhibitNewDownloads(ctx context.Context, in *proto_downloader.ProhibitNewDownloadsRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) { - return c.server.ProhibitNewDownloads(ctx, in) +func (c *DownloaderClient) Prohibit(ctx context.Context, in *proto_downloader.ProhibitRequest, opts ...grpc.CallOption) (*proto_downloader.ProhibitReply, error) { + return c.server.Prohibit(ctx, in) } func (c *DownloaderClient) Delete(ctx context.Context, in *proto_downloader.DeleteRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) { return c.server.Delete(ctx, in) diff --git a/erigon-lib/direct/sentry_client_mock.go b/erigon-lib/direct/sentry_client_mock.go index 198fd149175..bc5ab2f3f46 100644 --- a/erigon-lib/direct/sentry_client_mock.go +++ b/erigon-lib/direct/sentry_client_mock.go @@ -10,14 +10,14 @@ package direct import ( - "context" - "reflect" - - "github.com/ledgerwatch/erigon-lib/gointerfaces/sentry" - "github.com/ledgerwatch/erigon-lib/gointerfaces/types" - "go.uber.org/mock/gomock" - "google.golang.org/grpc" - "google.golang.org/protobuf/types/known/emptypb" + context "context" + reflect "reflect" + + sentry "github.com/ledgerwatch/erigon-lib/gointerfaces/sentry" + types "github.com/ledgerwatch/erigon-lib/gointerfaces/types" + gomock "go.uber.org/mock/gomock" + grpc "google.golang.org/grpc" + emptypb "google.golang.org/protobuf/types/known/emptypb" ) // MockSentryClient is a mock of SentryClient interface. diff --git a/erigon-lib/downloader/downloader_grpc_server.go b/erigon-lib/downloader/downloader_grpc_server.go index 6923c2db923..0ca7e2ec0e6 100644 --- a/erigon-lib/downloader/downloader_grpc_server.go +++ b/erigon-lib/downloader/downloader_grpc_server.go @@ -45,8 +45,9 @@ type GrpcServer struct { d *Downloader } -func (s *GrpcServer) ProhibitNewDownloads(ctx context.Context, req *proto_downloader.ProhibitNewDownloadsRequest) (*emptypb.Empty, error) { - return &emptypb.Empty{}, s.d.torrentFS.ProhibitNewDownloads(req.Type) +func (s *GrpcServer) ProhibitNewDownloads(ctx context.Context, req *proto_downloader.ProhibitRequest) (*proto_downloader.ProhibitReply, error) { + whitelist, err := s.d.torrentFS.ProhibitNewDownloads(req.WhitelistAdd, req.WhitelistRemove) + return &proto_downloader.ProhibitReply{Whitelist: whitelist}, err } // Erigon "download once" - means restart/upgrade/downgrade will not download files (and will be fast) diff --git a/erigon-lib/downloader/torrent_files.go b/erigon-lib/downloader/torrent_files.go index 026edd94b47..f29b0a33efa 100644 --- a/erigon-lib/downloader/torrent_files.go +++ b/erigon-lib/downloader/torrent_files.go @@ -3,7 +3,6 @@ package downloader import ( "encoding/json" "fmt" - "io" "os" "path/filepath" "strings" @@ -170,77 +169,83 @@ const ProhibitNewDownloadsFileName = "prohibit_new_downloads.lock" // Erigon "download once" - means restart/upgrade/downgrade will not download files (and will be fast) // After "download once" - Erigon will produce and seed new files // Downloader will able: seed new files (already existing on FS), download uncomplete parts of existing files (if Verify found some bad parts) -func (tf *AtomicTorrentFS) ProhibitNewDownloads(t string) error { +func (tf *AtomicTorrentFS) ProhibitNewDownloads(whitelistAdd, whitelistRemove []string) (whitelist []string, err error) { tf.lock.Lock() defer tf.lock.Unlock() - return tf.prohibitNewDownloads(t) + return tf.prohibitNewDownloads(whitelistAdd, whitelistRemove) } -func (tf *AtomicTorrentFS) prohibitNewDownloads(t string) error { +func (tf *AtomicTorrentFS) prohibitNewDownloads(whitelistAdd, whitelistRemove []string) (whitelist []string, err error) { fPath := filepath.Join(tf.dir, ProhibitNewDownloadsFileName) exist := dir.FileExist(fPath) - var prohibitedList []string + + var _currentWhiteList []string if exist { torrentListJsonBytes, err := os.ReadFile(fPath) if err != nil { - return fmt.Errorf("read file: %w", err) + return nil, fmt.Errorf("read file: %w", err) } if len(torrentListJsonBytes) > 0 { - if err := json.Unmarshal(torrentListJsonBytes, &prohibitedList); err != nil { - return fmt.Errorf("unmarshal: %w", err) + if err := json.Unmarshal(torrentListJsonBytes, &_currentWhiteList); err != nil { + return nil, fmt.Errorf("unmarshal: %w", err) } } } - if slices.Contains(prohibitedList, t) { - return nil + + whiteList := make([]string, 0, len(_currentWhiteList)) + for _, it := range _currentWhiteList { + if slices.Contains(whitelistRemove, it) { + continue + } + whiteList = append(whiteList, it) } - prohibitedList = append(prohibitedList, t) - prohibitedListJsonBytes, err := json.Marshal(prohibitedList) + for _, it := range whitelistAdd { + if slices.Contains(whiteList, it) { + whiteList = append(whiteList, it) + continue + } + } + slices.Sort(whiteList) + + whiteListBytes, err := json.Marshal(whiteList) if err != nil { - return fmt.Errorf("marshal: %w", err) + return _currentWhiteList, fmt.Errorf("marshal: %w", err) } - if err := dir.WriteFileWithFsync(fPath, prohibitedListJsonBytes, 0644); err != nil { - return fmt.Errorf("write: %w", err) + if err := dir.WriteFileWithFsync(fPath, whiteListBytes, 0644); err != nil { + return _currentWhiteList, fmt.Errorf("write: %w", err) } - return nil + return whiteList, nil } -func (tf *AtomicTorrentFS) NewDownloadsAreProhibited(name string) (bool, error) { +func (tf *AtomicTorrentFS) NewDownloadsAreProhibited(name string) (prohibited bool, err error) { tf.lock.Lock() defer tf.lock.Unlock() return tf.newDownloadsAreProhibited(name) } -func (tf *AtomicTorrentFS) newDownloadsAreProhibited(name string) (bool, error) { +func (tf *AtomicTorrentFS) newDownloadsAreProhibited(name string) (prohibited bool, err error) { fPath := filepath.Join(tf.dir, ProhibitNewDownloadsFileName) exists := dir.FileExist(fPath) - if !exists { + if !exists { // no .lock - means all allowed return false, nil } - f, err := os.OpenFile(fPath, os.O_RDONLY, 0644) - if err != nil { - return false, err - } - defer f.Close() - var prohibitedList []string - torrentListJsonBytes, err := io.ReadAll(f) + var whiteList []string + whiteListBytes, err := os.ReadFile(fPath) if err != nil { return false, fmt.Errorf("NewDownloadsAreProhibited: read file: %w", err) } - if exists && len(torrentListJsonBytes) == 0 { // backward compatibility: if .lock exists and empty - it means everything is prohibited - return true, nil - } - if len(torrentListJsonBytes) > 0 { - if err := json.Unmarshal(torrentListJsonBytes, &prohibitedList); err != nil { + if len(whiteListBytes) > 0 { + if err := json.Unmarshal(whiteListBytes, &whiteList); err != nil { return false, fmt.Errorf("NewDownloadsAreProhibited: unmarshal: %w", err) } } - for _, p := range prohibitedList { - if strings.Contains(name, p) { - return true, nil + + for _, whiteListedItem := range whiteList { + if strings.Contains(name, whiteListedItem) { + return false, nil } } - return false, nil + return true, nil } diff --git a/erigon-lib/downloader/torrent_files_test.go b/erigon-lib/downloader/torrent_files_test.go index a936f1f3970..789aa808c1a 100644 --- a/erigon-lib/downloader/torrent_files_test.go +++ b/erigon-lib/downloader/torrent_files_test.go @@ -41,8 +41,9 @@ func TestFSProhibitBackwardCompat(t *testing.T) { require.NoError(err) tf := NewAtomicTorrentFS(dirs.Snap) - err = tf.prohibitNewDownloads("transactions") //upgrade + wl, err := tf.prohibitNewDownloads([]string{"transactions"}, nil) //upgrade require.NoError(err) + require.Equal(err, []string{"transactions"}, wl) prohibited, err := tf.NewDownloadsAreProhibited("v1-004900-005000-headers.seg") require.NoError(err) diff --git a/erigon-lib/go.mod b/erigon-lib/go.mod index 3f13cfb766c..d857a5832f8 100644 --- a/erigon-lib/go.mod +++ b/erigon-lib/go.mod @@ -5,7 +5,7 @@ go 1.20 require ( github.com/erigontech/mdbx-go v0.27.24 github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20240417163500-185a51876901 - github.com/ledgerwatch/interfaces v0.0.0-20240425033437-5355fd37741e + github.com/ledgerwatch/interfaces v0.0.0-20240425034152-dda221776f08 github.com/ledgerwatch/log/v3 v3.9.0 github.com/ledgerwatch/secp256k1 v1.0.0 ) diff --git a/erigon-lib/go.sum b/erigon-lib/go.sum index 53b0f12fd56..0497ff1cfa1 100644 --- a/erigon-lib/go.sum +++ b/erigon-lib/go.sum @@ -258,8 +258,8 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20240417163500-185a51876901 h1:gAcI47OHnt/1e/APIV0093NVdviIfAnBUzFyybmKL1Q= github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20240417163500-185a51876901/go.mod h1:3AuPxZc85jkehh/HA9h8gabv5MSi3kb/ddtzBsTVJFo= -github.com/ledgerwatch/interfaces v0.0.0-20240425033437-5355fd37741e h1:NsjLpSpKotdzk61ne5pxCHSMJP+cRQEHZVRJp5FOsXI= -github.com/ledgerwatch/interfaces v0.0.0-20240425033437-5355fd37741e/go.mod h1:ugQv1QllJzBny3cKZKxUrSnykkjkBgm27eQM6dnGAcc= +github.com/ledgerwatch/interfaces v0.0.0-20240425034152-dda221776f08 h1:NQRyMIGIapAFnr7hAY0xXQZPMBjtYCUAQ0UF1/saBaE= +github.com/ledgerwatch/interfaces v0.0.0-20240425034152-dda221776f08/go.mod h1:ugQv1QllJzBny3cKZKxUrSnykkjkBgm27eQM6dnGAcc= github.com/ledgerwatch/log/v3 v3.9.0 h1:iDwrXe0PVwBC68Dd94YSsHbMgQ3ufsgjzXtFNFVZFRk= github.com/ledgerwatch/log/v3 v3.9.0/go.mod h1:EiAY6upmI/6LkNhOVxb4eVsmsP11HZCnZ3PlJMjYiqE= github.com/ledgerwatch/secp256k1 v1.0.0 h1:Usvz87YoTG0uePIV8woOof5cQnLXGYa162rFf3YnwaQ= diff --git a/erigon-lib/gointerfaces/downloader/downloader.pb.go b/erigon-lib/gointerfaces/downloader/downloader.pb.go index dec9c5cc3e7..3761a45a6b1 100644 --- a/erigon-lib/gointerfaces/downloader/downloader.pb.go +++ b/erigon-lib/gointerfaces/downloader/downloader.pb.go @@ -251,16 +251,17 @@ func (*StatsRequest) Descriptor() ([]byte, []int) { return file_downloader_downloader_proto_rawDescGZIP(), []int{4} } -type ProhibitNewDownloadsRequest struct { +type ProhibitRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Type string `protobuf:"bytes,1,opt,name=type,proto3" json:"type,omitempty"` + WhitelistAdd []string `protobuf:"bytes,1,rep,name=whitelistAdd,proto3" json:"whitelistAdd,omitempty"` // nil - means "don't modify". non-nil - means "merge with current whitelist". + WhitelistRemove []string `protobuf:"bytes,2,rep,name=whitelistRemove,proto3" json:"whitelistRemove,omitempty"` // nil - means "don't modify" } -func (x *ProhibitNewDownloadsRequest) Reset() { - *x = ProhibitNewDownloadsRequest{} +func (x *ProhibitRequest) Reset() { + *x = ProhibitRequest{} if protoimpl.UnsafeEnabled { mi := &file_downloader_downloader_proto_msgTypes[5] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -268,13 +269,13 @@ func (x *ProhibitNewDownloadsRequest) Reset() { } } -func (x *ProhibitNewDownloadsRequest) String() string { +func (x *ProhibitRequest) String() string { return protoimpl.X.MessageStringOf(x) } -func (*ProhibitNewDownloadsRequest) ProtoMessage() {} +func (*ProhibitRequest) ProtoMessage() {} -func (x *ProhibitNewDownloadsRequest) ProtoReflect() protoreflect.Message { +func (x *ProhibitRequest) ProtoReflect() protoreflect.Message { mi := &file_downloader_downloader_proto_msgTypes[5] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -286,16 +287,70 @@ func (x *ProhibitNewDownloadsRequest) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use ProhibitNewDownloadsRequest.ProtoReflect.Descriptor instead. -func (*ProhibitNewDownloadsRequest) Descriptor() ([]byte, []int) { +// Deprecated: Use ProhibitRequest.ProtoReflect.Descriptor instead. +func (*ProhibitRequest) Descriptor() ([]byte, []int) { return file_downloader_downloader_proto_rawDescGZIP(), []int{5} } -func (x *ProhibitNewDownloadsRequest) GetType() string { +func (x *ProhibitRequest) GetWhitelistAdd() []string { if x != nil { - return x.Type + return x.WhitelistAdd } - return "" + return nil +} + +func (x *ProhibitRequest) GetWhitelistRemove() []string { + if x != nil { + return x.WhitelistRemove + } + return nil +} + +type ProhibitReply struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Whitelist []string `protobuf:"bytes,1,rep,name=whitelist,proto3" json:"whitelist,omitempty"` // current whitelist +} + +func (x *ProhibitReply) Reset() { + *x = ProhibitReply{} + if protoimpl.UnsafeEnabled { + mi := &file_downloader_downloader_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ProhibitReply) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ProhibitReply) ProtoMessage() {} + +func (x *ProhibitReply) ProtoReflect() protoreflect.Message { + mi := &file_downloader_downloader_proto_msgTypes[6] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ProhibitReply.ProtoReflect.Descriptor instead. +func (*ProhibitReply) Descriptor() ([]byte, []int) { + return file_downloader_downloader_proto_rawDescGZIP(), []int{6} +} + +func (x *ProhibitReply) GetWhitelist() []string { + if x != nil { + return x.Whitelist + } + return nil } type StatsReply struct { @@ -323,7 +378,7 @@ type StatsReply struct { func (x *StatsReply) Reset() { *x = StatsReply{} if protoimpl.UnsafeEnabled { - mi := &file_downloader_downloader_proto_msgTypes[6] + mi := &file_downloader_downloader_proto_msgTypes[7] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -336,7 +391,7 @@ func (x *StatsReply) String() string { func (*StatsReply) ProtoMessage() {} func (x *StatsReply) ProtoReflect() protoreflect.Message { - mi := &file_downloader_downloader_proto_msgTypes[6] + mi := &file_downloader_downloader_proto_msgTypes[7] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -349,7 +404,7 @@ func (x *StatsReply) ProtoReflect() protoreflect.Message { // Deprecated: Use StatsReply.ProtoReflect.Descriptor instead. func (*StatsReply) Descriptor() ([]byte, []int) { - return file_downloader_downloader_proto_rawDescGZIP(), []int{6} + return file_downloader_downloader_proto_rawDescGZIP(), []int{7} } func (x *StatsReply) GetMetadataReady() int32 { @@ -443,57 +498,62 @@ var file_downloader_downloader_proto_rawDesc = []byte{ 0x73, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x70, 0x61, 0x74, 0x68, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x05, 0x70, 0x61, 0x74, 0x68, 0x73, 0x22, 0x0f, 0x0a, 0x0d, 0x56, 0x65, 0x72, 0x69, 0x66, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x0e, 0x0a, 0x0c, 0x53, 0x74, 0x61, - 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x31, 0x0a, 0x1b, 0x50, 0x72, 0x6f, - 0x68, 0x69, 0x62, 0x69, 0x74, 0x4e, 0x65, 0x77, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, - 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x22, 0xee, 0x02, 0x0a, - 0x0a, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x12, 0x25, 0x0a, 0x0e, 0x6d, - 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x72, 0x65, 0x61, 0x64, 0x79, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x05, 0x52, 0x0d, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x65, 0x61, - 0x64, 0x79, 0x12, 0x1f, 0x0a, 0x0b, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x5f, 0x74, 0x6f, 0x74, 0x61, - 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0a, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x54, 0x6f, - 0x74, 0x61, 0x6c, 0x12, 0x21, 0x0a, 0x0c, 0x70, 0x65, 0x65, 0x72, 0x73, 0x5f, 0x75, 0x6e, 0x69, - 0x71, 0x75, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0b, 0x70, 0x65, 0x65, 0x72, 0x73, - 0x55, 0x6e, 0x69, 0x71, 0x75, 0x65, 0x12, 0x2b, 0x0a, 0x11, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, - 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x5f, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x18, 0x05, 0x20, 0x01, 0x28, - 0x04, 0x52, 0x10, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x54, 0x6f, - 0x74, 0x61, 0x6c, 0x12, 0x1c, 0x0a, 0x09, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, - 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, - 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x18, 0x07, 0x20, - 0x01, 0x28, 0x02, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x12, 0x27, 0x0a, - 0x0f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x5f, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, - 0x18, 0x08, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0e, 0x62, 0x79, 0x74, 0x65, 0x73, 0x43, 0x6f, 0x6d, - 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x12, 0x1f, 0x0a, 0x0b, 0x62, 0x79, 0x74, 0x65, 0x73, 0x5f, - 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x18, 0x09, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0a, 0x62, 0x79, 0x74, - 0x65, 0x73, 0x54, 0x6f, 0x74, 0x61, 0x6c, 0x12, 0x1f, 0x0a, 0x0b, 0x75, 0x70, 0x6c, 0x6f, 0x61, - 0x64, 0x5f, 0x72, 0x61, 0x74, 0x65, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0a, 0x75, 0x70, - 0x6c, 0x6f, 0x61, 0x64, 0x52, 0x61, 0x74, 0x65, 0x12, 0x23, 0x0a, 0x0d, 0x64, 0x6f, 0x77, 0x6e, - 0x6c, 0x6f, 0x61, 0x64, 0x5f, 0x72, 0x61, 0x74, 0x65, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x04, 0x52, - 0x0c, 0x64, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x52, 0x61, 0x74, 0x65, 0x32, 0xdb, 0x02, - 0x0a, 0x0a, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x65, 0x72, 0x12, 0x59, 0x0a, 0x14, - 0x50, 0x72, 0x6f, 0x68, 0x69, 0x62, 0x69, 0x74, 0x4e, 0x65, 0x77, 0x44, 0x6f, 0x77, 0x6e, 0x6c, - 0x6f, 0x61, 0x64, 0x73, 0x12, 0x27, 0x2e, 0x64, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x65, - 0x72, 0x2e, 0x50, 0x72, 0x6f, 0x68, 0x69, 0x62, 0x69, 0x74, 0x4e, 0x65, 0x77, 0x44, 0x6f, 0x77, - 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, + 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x5f, 0x0a, 0x0f, 0x50, 0x72, 0x6f, + 0x68, 0x69, 0x62, 0x69, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x22, 0x0a, 0x0c, + 0x77, 0x68, 0x69, 0x74, 0x65, 0x6c, 0x69, 0x73, 0x74, 0x41, 0x64, 0x64, 0x18, 0x01, 0x20, 0x03, + 0x28, 0x09, 0x52, 0x0c, 0x77, 0x68, 0x69, 0x74, 0x65, 0x6c, 0x69, 0x73, 0x74, 0x41, 0x64, 0x64, + 0x12, 0x28, 0x0a, 0x0f, 0x77, 0x68, 0x69, 0x74, 0x65, 0x6c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x6d, + 0x6f, 0x76, 0x65, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0f, 0x77, 0x68, 0x69, 0x74, 0x65, + 0x6c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x22, 0x2d, 0x0a, 0x0d, 0x50, 0x72, + 0x6f, 0x68, 0x69, 0x62, 0x69, 0x74, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x12, 0x1c, 0x0a, 0x09, 0x77, + 0x68, 0x69, 0x74, 0x65, 0x6c, 0x69, 0x73, 0x74, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x09, + 0x77, 0x68, 0x69, 0x74, 0x65, 0x6c, 0x69, 0x73, 0x74, 0x22, 0xee, 0x02, 0x0a, 0x0a, 0x53, 0x74, + 0x61, 0x74, 0x73, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x12, 0x25, 0x0a, 0x0e, 0x6d, 0x65, 0x74, 0x61, + 0x64, 0x61, 0x74, 0x61, 0x5f, 0x72, 0x65, 0x61, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, + 0x52, 0x0d, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x65, 0x61, 0x64, 0x79, 0x12, + 0x1f, 0x0a, 0x0b, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x5f, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x05, 0x52, 0x0a, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x54, 0x6f, 0x74, 0x61, 0x6c, + 0x12, 0x21, 0x0a, 0x0c, 0x70, 0x65, 0x65, 0x72, 0x73, 0x5f, 0x75, 0x6e, 0x69, 0x71, 0x75, 0x65, + 0x18, 0x04, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0b, 0x70, 0x65, 0x65, 0x72, 0x73, 0x55, 0x6e, 0x69, + 0x71, 0x75, 0x65, 0x12, 0x2b, 0x0a, 0x11, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, + 0x6e, 0x73, 0x5f, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x18, 0x05, 0x20, 0x01, 0x28, 0x04, 0x52, 0x10, + 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x54, 0x6f, 0x74, 0x61, 0x6c, + 0x12, 0x1c, 0x0a, 0x09, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x18, 0x06, 0x20, + 0x01, 0x28, 0x08, 0x52, 0x09, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x12, 0x1a, + 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x18, 0x07, 0x20, 0x01, 0x28, 0x02, + 0x52, 0x08, 0x70, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x12, 0x27, 0x0a, 0x0f, 0x62, 0x79, + 0x74, 0x65, 0x73, 0x5f, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x18, 0x08, 0x20, + 0x01, 0x28, 0x04, 0x52, 0x0e, 0x62, 0x79, 0x74, 0x65, 0x73, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, + 0x74, 0x65, 0x64, 0x12, 0x1f, 0x0a, 0x0b, 0x62, 0x79, 0x74, 0x65, 0x73, 0x5f, 0x74, 0x6f, 0x74, + 0x61, 0x6c, 0x18, 0x09, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0a, 0x62, 0x79, 0x74, 0x65, 0x73, 0x54, + 0x6f, 0x74, 0x61, 0x6c, 0x12, 0x1f, 0x0a, 0x0b, 0x75, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x5f, 0x72, + 0x61, 0x74, 0x65, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0a, 0x75, 0x70, 0x6c, 0x6f, 0x61, + 0x64, 0x52, 0x61, 0x74, 0x65, 0x12, 0x23, 0x0a, 0x0d, 0x64, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, + 0x64, 0x5f, 0x72, 0x61, 0x74, 0x65, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0c, 0x64, 0x6f, + 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x52, 0x61, 0x74, 0x65, 0x32, 0xc6, 0x02, 0x0a, 0x0a, 0x44, + 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x65, 0x72, 0x12, 0x44, 0x0a, 0x08, 0x50, 0x72, 0x6f, + 0x68, 0x69, 0x62, 0x69, 0x74, 0x12, 0x1b, 0x2e, 0x64, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, + 0x65, 0x72, 0x2e, 0x50, 0x72, 0x6f, 0x68, 0x69, 0x62, 0x69, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x64, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x65, 0x72, 0x2e, + 0x50, 0x72, 0x6f, 0x68, 0x69, 0x62, 0x69, 0x74, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, + 0x37, 0x0a, 0x03, 0x41, 0x64, 0x64, 0x12, 0x16, 0x2e, 0x64, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, + 0x64, 0x65, 0x72, 0x2e, 0x41, 0x64, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, + 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, + 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x3d, 0x0a, 0x06, 0x44, 0x65, 0x6c, 0x65, + 0x74, 0x65, 0x12, 0x19, 0x2e, 0x64, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x65, 0x72, 0x2e, + 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, - 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x37, 0x0a, 0x03, 0x41, 0x64, 0x64, 0x12, 0x16, - 0x2e, 0x64, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x65, 0x72, 0x2e, 0x41, 0x64, 0x64, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, - 0x12, 0x3d, 0x0a, 0x06, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x12, 0x19, 0x2e, 0x64, 0x6f, 0x77, - 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x65, 0x72, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, - 0x3d, 0x0a, 0x06, 0x56, 0x65, 0x72, 0x69, 0x66, 0x79, 0x12, 0x19, 0x2e, 0x64, 0x6f, 0x77, 0x6e, - 0x6c, 0x6f, 0x61, 0x64, 0x65, 0x72, 0x2e, 0x56, 0x65, 0x72, 0x69, 0x66, 0x79, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x3b, - 0x0a, 0x05, 0x53, 0x74, 0x61, 0x74, 0x73, 0x12, 0x18, 0x2e, 0x64, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, - 0x61, 0x64, 0x65, 0x72, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x16, 0x2e, 0x64, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x65, 0x72, 0x2e, 0x53, - 0x74, 0x61, 0x74, 0x73, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x42, 0x19, 0x5a, 0x17, 0x2e, - 0x2f, 0x64, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x65, 0x72, 0x3b, 0x64, 0x6f, 0x77, 0x6e, - 0x6c, 0x6f, 0x61, 0x64, 0x65, 0x72, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x3d, 0x0a, 0x06, 0x56, 0x65, 0x72, 0x69, 0x66, + 0x79, 0x12, 0x19, 0x2e, 0x64, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x65, 0x72, 0x2e, 0x56, + 0x65, 0x72, 0x69, 0x66, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, + 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, + 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x3b, 0x0a, 0x05, 0x53, 0x74, 0x61, 0x74, 0x73, 0x12, + 0x18, 0x2e, 0x64, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x65, 0x72, 0x2e, 0x53, 0x74, 0x61, + 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x64, 0x6f, 0x77, 0x6e, + 0x6c, 0x6f, 0x61, 0x64, 0x65, 0x72, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x65, 0x70, 0x6c, + 0x79, 0x22, 0x00, 0x42, 0x19, 0x5a, 0x17, 0x2e, 0x2f, 0x64, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, + 0x64, 0x65, 0x72, 0x3b, 0x64, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x65, 0x72, 0x62, 0x06, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -508,31 +568,32 @@ func file_downloader_downloader_proto_rawDescGZIP() []byte { return file_downloader_downloader_proto_rawDescData } -var file_downloader_downloader_proto_msgTypes = make([]protoimpl.MessageInfo, 7) +var file_downloader_downloader_proto_msgTypes = make([]protoimpl.MessageInfo, 8) var file_downloader_downloader_proto_goTypes = []interface{}{ - (*AddItem)(nil), // 0: downloader.AddItem - (*AddRequest)(nil), // 1: downloader.AddRequest - (*DeleteRequest)(nil), // 2: downloader.DeleteRequest - (*VerifyRequest)(nil), // 3: downloader.VerifyRequest - (*StatsRequest)(nil), // 4: downloader.StatsRequest - (*ProhibitNewDownloadsRequest)(nil), // 5: downloader.ProhibitNewDownloadsRequest - (*StatsReply)(nil), // 6: downloader.StatsReply - (*types.H160)(nil), // 7: types.H160 - (*emptypb.Empty)(nil), // 8: google.protobuf.Empty + (*AddItem)(nil), // 0: downloader.AddItem + (*AddRequest)(nil), // 1: downloader.AddRequest + (*DeleteRequest)(nil), // 2: downloader.DeleteRequest + (*VerifyRequest)(nil), // 3: downloader.VerifyRequest + (*StatsRequest)(nil), // 4: downloader.StatsRequest + (*ProhibitRequest)(nil), // 5: downloader.ProhibitRequest + (*ProhibitReply)(nil), // 6: downloader.ProhibitReply + (*StatsReply)(nil), // 7: downloader.StatsReply + (*types.H160)(nil), // 8: types.H160 + (*emptypb.Empty)(nil), // 9: google.protobuf.Empty } var file_downloader_downloader_proto_depIdxs = []int32{ - 7, // 0: downloader.AddItem.torrent_hash:type_name -> types.H160 + 8, // 0: downloader.AddItem.torrent_hash:type_name -> types.H160 0, // 1: downloader.AddRequest.items:type_name -> downloader.AddItem - 5, // 2: downloader.Downloader.ProhibitNewDownloads:input_type -> downloader.ProhibitNewDownloadsRequest + 5, // 2: downloader.Downloader.Prohibit:input_type -> downloader.ProhibitRequest 1, // 3: downloader.Downloader.Add:input_type -> downloader.AddRequest 2, // 4: downloader.Downloader.Delete:input_type -> downloader.DeleteRequest 3, // 5: downloader.Downloader.Verify:input_type -> downloader.VerifyRequest 4, // 6: downloader.Downloader.Stats:input_type -> downloader.StatsRequest - 8, // 7: downloader.Downloader.ProhibitNewDownloads:output_type -> google.protobuf.Empty - 8, // 8: downloader.Downloader.Add:output_type -> google.protobuf.Empty - 8, // 9: downloader.Downloader.Delete:output_type -> google.protobuf.Empty - 8, // 10: downloader.Downloader.Verify:output_type -> google.protobuf.Empty - 6, // 11: downloader.Downloader.Stats:output_type -> downloader.StatsReply + 6, // 7: downloader.Downloader.Prohibit:output_type -> downloader.ProhibitReply + 9, // 8: downloader.Downloader.Add:output_type -> google.protobuf.Empty + 9, // 9: downloader.Downloader.Delete:output_type -> google.protobuf.Empty + 9, // 10: downloader.Downloader.Verify:output_type -> google.protobuf.Empty + 7, // 11: downloader.Downloader.Stats:output_type -> downloader.StatsReply 7, // [7:12] is the sub-list for method output_type 2, // [2:7] is the sub-list for method input_type 2, // [2:2] is the sub-list for extension type_name @@ -607,7 +668,7 @@ func file_downloader_downloader_proto_init() { } } file_downloader_downloader_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ProhibitNewDownloadsRequest); i { + switch v := v.(*ProhibitRequest); i { case 0: return &v.state case 1: @@ -619,6 +680,18 @@ func file_downloader_downloader_proto_init() { } } file_downloader_downloader_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ProhibitReply); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_downloader_downloader_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*StatsReply); i { case 0: return &v.state @@ -637,7 +710,7 @@ func file_downloader_downloader_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_downloader_downloader_proto_rawDesc, NumEnums: 0, - NumMessages: 7, + NumMessages: 8, NumExtensions: 0, NumServices: 1, }, diff --git a/erigon-lib/gointerfaces/downloader/downloader_grpc.pb.go b/erigon-lib/gointerfaces/downloader/downloader_grpc.pb.go index 369c9b494c4..5a70ffb6656 100644 --- a/erigon-lib/gointerfaces/downloader/downloader_grpc.pb.go +++ b/erigon-lib/gointerfaces/downloader/downloader_grpc.pb.go @@ -20,11 +20,11 @@ import ( const _ = grpc.SupportPackageIsVersion7 const ( - Downloader_ProhibitNewDownloads_FullMethodName = "/downloader.Downloader/ProhibitNewDownloads" - Downloader_Add_FullMethodName = "/downloader.Downloader/Add" - Downloader_Delete_FullMethodName = "/downloader.Downloader/Delete" - Downloader_Verify_FullMethodName = "/downloader.Downloader/Verify" - Downloader_Stats_FullMethodName = "/downloader.Downloader/Stats" + Downloader_Prohibit_FullMethodName = "/downloader.Downloader/Prohibit" + Downloader_Add_FullMethodName = "/downloader.Downloader/Add" + Downloader_Delete_FullMethodName = "/downloader.Downloader/Delete" + Downloader_Verify_FullMethodName = "/downloader.Downloader/Verify" + Downloader_Stats_FullMethodName = "/downloader.Downloader/Stats" ) // DownloaderClient is the client API for Downloader service. @@ -33,8 +33,12 @@ const ( type DownloaderClient interface { // Erigon "download once" - means restart/upgrade/downgrade will not download files (and will be fast) // After "download once" - Erigon will produce and seed new files - // Downloader will able: seed new files (already existing on FS), download uncomplete parts of existing files (if Verify found some bad parts) - ProhibitNewDownloads(ctx context.Context, in *ProhibitNewDownloadsRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) + // After `ProhibitNew` call - downloader stil will able: + // - seed new files (already existing on FS) + // - download uncomplete parts of existing files (if Verify found some bad parts) + // + // `ProhibitNew` what whitelist based on file-type - can add remove items there. + Prohibit(ctx context.Context, in *ProhibitRequest, opts ...grpc.CallOption) (*ProhibitReply, error) // Adding new file to downloader: non-existing files it will download, existing - seed Add(ctx context.Context, in *AddRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) Delete(ctx context.Context, in *DeleteRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) @@ -52,9 +56,9 @@ func NewDownloaderClient(cc grpc.ClientConnInterface) DownloaderClient { return &downloaderClient{cc} } -func (c *downloaderClient) ProhibitNewDownloads(ctx context.Context, in *ProhibitNewDownloadsRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) { - out := new(emptypb.Empty) - err := c.cc.Invoke(ctx, Downloader_ProhibitNewDownloads_FullMethodName, in, out, opts...) +func (c *downloaderClient) Prohibit(ctx context.Context, in *ProhibitRequest, opts ...grpc.CallOption) (*ProhibitReply, error) { + out := new(ProhibitReply) + err := c.cc.Invoke(ctx, Downloader_Prohibit_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -103,8 +107,12 @@ func (c *downloaderClient) Stats(ctx context.Context, in *StatsRequest, opts ... type DownloaderServer interface { // Erigon "download once" - means restart/upgrade/downgrade will not download files (and will be fast) // After "download once" - Erigon will produce and seed new files - // Downloader will able: seed new files (already existing on FS), download uncomplete parts of existing files (if Verify found some bad parts) - ProhibitNewDownloads(context.Context, *ProhibitNewDownloadsRequest) (*emptypb.Empty, error) + // After `ProhibitNew` call - downloader stil will able: + // - seed new files (already existing on FS) + // - download uncomplete parts of existing files (if Verify found some bad parts) + // + // `ProhibitNew` what whitelist based on file-type - can add remove items there. + Prohibit(context.Context, *ProhibitRequest) (*ProhibitReply, error) // Adding new file to downloader: non-existing files it will download, existing - seed Add(context.Context, *AddRequest) (*emptypb.Empty, error) Delete(context.Context, *DeleteRequest) (*emptypb.Empty, error) @@ -119,8 +127,8 @@ type DownloaderServer interface { type UnimplementedDownloaderServer struct { } -func (UnimplementedDownloaderServer) ProhibitNewDownloads(context.Context, *ProhibitNewDownloadsRequest) (*emptypb.Empty, error) { - return nil, status.Errorf(codes.Unimplemented, "method ProhibitNewDownloads not implemented") +func (UnimplementedDownloaderServer) Prohibit(context.Context, *ProhibitRequest) (*ProhibitReply, error) { + return nil, status.Errorf(codes.Unimplemented, "method Prohibit not implemented") } func (UnimplementedDownloaderServer) Add(context.Context, *AddRequest) (*emptypb.Empty, error) { return nil, status.Errorf(codes.Unimplemented, "method Add not implemented") @@ -147,20 +155,20 @@ func RegisterDownloaderServer(s grpc.ServiceRegistrar, srv DownloaderServer) { s.RegisterService(&Downloader_ServiceDesc, srv) } -func _Downloader_ProhibitNewDownloads_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(ProhibitNewDownloadsRequest) +func _Downloader_Prohibit_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ProhibitRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { - return srv.(DownloaderServer).ProhibitNewDownloads(ctx, in) + return srv.(DownloaderServer).Prohibit(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: Downloader_ProhibitNewDownloads_FullMethodName, + FullMethod: Downloader_Prohibit_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(DownloaderServer).ProhibitNewDownloads(ctx, req.(*ProhibitNewDownloadsRequest)) + return srv.(DownloaderServer).Prohibit(ctx, req.(*ProhibitRequest)) } return interceptor(ctx, in, info, handler) } @@ -245,8 +253,8 @@ var Downloader_ServiceDesc = grpc.ServiceDesc{ HandlerType: (*DownloaderServer)(nil), Methods: []grpc.MethodDesc{ { - MethodName: "ProhibitNewDownloads", - Handler: _Downloader_ProhibitNewDownloads_Handler, + MethodName: "Prohibit", + Handler: _Downloader_Prohibit_Handler, }, { MethodName: "Add", diff --git a/turbo/snapshotsync/snapshotsync.go b/turbo/snapshotsync/snapshotsync.go index 334fe592d0d..a1406fab152 100644 --- a/turbo/snapshotsync/snapshotsync.go +++ b/turbo/snapshotsync/snapshotsync.go @@ -196,29 +196,25 @@ func WaitForDownloader(ctx context.Context, logPrefix string, histV3, blobs bool return err } - // ProhibitNewDownloads implies - so only make the download request once, - // - // Erigon "download once" - means restart/upgrade/downgrade will not download files (and will be fast) + // Erigon has "download once" invariant - means restart/upgrade/downgrade will not download new files (and will be fast) // After "download once" - Erigon will produce and seed new files - // Downloader will able: seed new files (already existing on FS), download uncomplete parts of existing files (if Verify found some bad parts) - // - // after the initial call the downloader or snapshot-lock.file will prevent this download from running + // After `Prohibit` call - downloader stil will able: + // - seed new (generated by Erigon) files + // - seed existing on Disk files + // - download uncomplete parts of existing on Disk files (if Verify found some bad parts) // - - // prohibits further downloads, except some exceptions - for _, p := range snaptype.AllTypes { - if (p.Enum() == snaptype.BeaconBlocks.Enum() || p.Enum() == snaptype.BlobSidecars.Enum()) && caplin == NoCaplin { - continue - } - if p.Enum() == snaptype.BlobSidecars.Enum() && !blobs { - continue - } - if _, err := snapshotDownloader.ProhibitNewDownloads(ctx, &proto_downloader.ProhibitNewDownloadsRequest{ - Type: p.String(), - }); err != nil { - return err + // Caplin's code is ready for backgrond 1-time download of it's files. + // So, we allow users of Caplin download and index new type of files - in background. + var whitelist []string + if caplin != NoCaplin { + whitelist = append(whitelist, snaptype.BeaconBlocks.Enum().String()) + if blobs { + whitelist = append(whitelist, snaptype.BlobSidecars.Enum().String()) } } + if _, err := snapshotDownloader.Prohibit(ctx, &proto_downloader.ProhibitRequest{WhitelistAdd: whitelist}); err != nil { + return err + } if err := rawdb.WriteSnapshots(tx, blockReader.FrozenFiles(), agg.Files()); err != nil { return err From 5e6d128ed83d3b7747ddd820f005cd5459399a99 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Thu, 25 Apr 2024 11:19:32 +0700 Subject: [PATCH 09/14] save --- erigon-lib/downloader/torrent_files.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/erigon-lib/downloader/torrent_files.go b/erigon-lib/downloader/torrent_files.go index f29b0a33efa..479fcabe438 100644 --- a/erigon-lib/downloader/torrent_files.go +++ b/erigon-lib/downloader/torrent_files.go @@ -168,7 +168,12 @@ const ProhibitNewDownloadsFileName = "prohibit_new_downloads.lock" // Erigon "download once" - means restart/upgrade/downgrade will not download files (and will be fast) // After "download once" - Erigon will produce and seed new files -// Downloader will able: seed new files (already existing on FS), download uncomplete parts of existing files (if Verify found some bad parts) +// After `Prohibit` call - downloader stil will able: +// - seed new (generated by Erigon) files +// - seed existing on Disk files +// - download uncomplete parts of existing on Disk files (if Verify found some bad parts) +// +// `Prohibit` has `whitelist` feature - based on file-type func (tf *AtomicTorrentFS) ProhibitNewDownloads(whitelistAdd, whitelistRemove []string) (whitelist []string, err error) { tf.lock.Lock() defer tf.lock.Unlock() From 1b8d926f6e6f80286fc190602a26d517b81c2924 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Thu, 25 Apr 2024 11:27:52 +0700 Subject: [PATCH 10/14] save --- eth/backend.go | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/eth/backend.go b/eth/backend.go index 2acd655f9e5..28ee59e5f03 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -55,7 +55,6 @@ import ( "github.com/ledgerwatch/erigon-lib/downloader" "github.com/ledgerwatch/erigon-lib/downloader/downloadercfg" "github.com/ledgerwatch/erigon-lib/downloader/downloadergrpc" - "github.com/ledgerwatch/erigon-lib/downloader/snaptype" protodownloader "github.com/ledgerwatch/erigon-lib/gointerfaces/downloader" "github.com/ledgerwatch/erigon-lib/gointerfaces/grpcutil" "github.com/ledgerwatch/erigon-lib/gointerfaces/remote" @@ -786,11 +785,7 @@ func New(ctx context.Context, stack *node.Node, config *ethconfig.Config, logger hook := stages2.NewHook(backend.sentryCtx, backend.chainDB, backend.notifications, backend.stagedSync, backend.blockReader, backend.chainConfig, backend.logger, backend.sentriesClient.SetStatus) if !config.Sync.UseSnapshots && backend.downloaderClient != nil { - for _, p := range snaptype.AllTypes { - backend.downloaderClient.ProhibitNewDownloads(ctx, &protodownloader.ProhibitNewDownloadsRequest{ - Type: p.String(), - }) - } + _, _ = backend.downloaderClient.Prohibit(ctx, &protodownloader.ProhibitRequest{}) } checkStateRoot := true From 8a32a24e13265542df29661acf83f497d61a4261 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Thu, 25 Apr 2024 12:48:17 +0700 Subject: [PATCH 11/14] save --- erigon-lib/downloader/downloader_grpc_server.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/erigon-lib/downloader/downloader_grpc_server.go b/erigon-lib/downloader/downloader_grpc_server.go index 0ca7e2ec0e6..dea7e0dc769 100644 --- a/erigon-lib/downloader/downloader_grpc_server.go +++ b/erigon-lib/downloader/downloader_grpc_server.go @@ -45,7 +45,7 @@ type GrpcServer struct { d *Downloader } -func (s *GrpcServer) ProhibitNewDownloads(ctx context.Context, req *proto_downloader.ProhibitRequest) (*proto_downloader.ProhibitReply, error) { +func (s *GrpcServer) Prohibit(ctx context.Context, req *proto_downloader.ProhibitRequest) (*proto_downloader.ProhibitReply, error) { whitelist, err := s.d.torrentFS.ProhibitNewDownloads(req.WhitelistAdd, req.WhitelistRemove) return &proto_downloader.ProhibitReply{Whitelist: whitelist}, err } @@ -59,10 +59,6 @@ func (s *GrpcServer) Add(ctx context.Context, request *proto_downloader.AddReque logEvery := time.NewTicker(20 * time.Second) defer logEvery.Stop() - s.d.lock.Lock() - s.d.stats.Requested += len(request.Items) - s.d.lock.Unlock() - for i, it := range request.Items { if it.Path == "" { return nil, fmt.Errorf("field 'path' is required") From 7162b4399a1ccba6e5420214e9f52c30100c1299 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Thu, 25 Apr 2024 16:06:16 +0700 Subject: [PATCH 12/14] save --- erigon-lib/downloader/torrent_files_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/erigon-lib/downloader/torrent_files_test.go b/erigon-lib/downloader/torrent_files_test.go index 789aa808c1a..6fe2334c616 100644 --- a/erigon-lib/downloader/torrent_files_test.go +++ b/erigon-lib/downloader/torrent_files_test.go @@ -43,7 +43,7 @@ func TestFSProhibitBackwardCompat(t *testing.T) { tf := NewAtomicTorrentFS(dirs.Snap) wl, err := tf.prohibitNewDownloads([]string{"transactions"}, nil) //upgrade require.NoError(err) - require.Equal(err, []string{"transactions"}, wl) + require.Equal([]string{"transactions"}, wl) prohibited, err := tf.NewDownloadsAreProhibited("v1-004900-005000-headers.seg") require.NoError(err) From abad420588d02fe828e990201e529348b7634ba0 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Thu, 25 Apr 2024 16:09:46 +0700 Subject: [PATCH 13/14] save --- erigon-lib/downloader/torrent_files.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/erigon-lib/downloader/torrent_files.go b/erigon-lib/downloader/torrent_files.go index 479fcabe438..208af32d39e 100644 --- a/erigon-lib/downloader/torrent_files.go +++ b/erigon-lib/downloader/torrent_files.go @@ -198,6 +198,7 @@ func (tf *AtomicTorrentFS) prohibitNewDownloads(whitelistAdd, whitelistRemove [] } whiteList := make([]string, 0, len(_currentWhiteList)) + // copy all item except delted for _, it := range _currentWhiteList { if slices.Contains(whitelistRemove, it) { continue @@ -205,10 +206,10 @@ func (tf *AtomicTorrentFS) prohibitNewDownloads(whitelistAdd, whitelistRemove [] whiteList = append(whiteList, it) } + // add all new whitelisted items for _, it := range whitelistAdd { - if slices.Contains(whiteList, it) { + if !slices.Contains(whiteList, it) { whiteList = append(whiteList, it) - continue } } slices.Sort(whiteList) From a58aae8394861823f22bd5faed7499daae5464bf Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Thu, 25 Apr 2024 16:10:29 +0700 Subject: [PATCH 14/14] save --- erigon-lib/downloader/torrent_files_test.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/erigon-lib/downloader/torrent_files_test.go b/erigon-lib/downloader/torrent_files_test.go index 6fe2334c616..6be124b6ce0 100644 --- a/erigon-lib/downloader/torrent_files_test.go +++ b/erigon-lib/downloader/torrent_files_test.go @@ -47,9 +47,13 @@ func TestFSProhibitBackwardCompat(t *testing.T) { prohibited, err := tf.NewDownloadsAreProhibited("v1-004900-005000-headers.seg") require.NoError(err) - require.False(prohibited) + require.True(prohibited) prohibited, err = tf.NewDownloadsAreProhibited("v1-004900-005000-headers.seg.torrent") require.NoError(err) + require.True(prohibited) + + prohibited, err = tf.NewDownloadsAreProhibited("v1-004900-005000-transactions.seg") + require.NoError(err) require.False(prohibited) }) }