From 662f1f70e7e39148b329f38b9b828240380c6bc2 Mon Sep 17 00:00:00 2001 From: gammazero <11790789+gammazero@users.noreply.github.com> Date: Fri, 20 Sep 2024 12:00:20 -0700 Subject: [PATCH 1/5] Add ability to configure bitswap wanthave replacement A new flag and environ var is available to specify the maximum blocksize that bitswap will replace WantHave with WantBlock responses. A value of zero diables replacement and avoids block size lookup for WantHave requests. This option only applies when bitswap is enabled. --- go.mod | 2 +- go.sum | 4 ++-- main.go | 7 +++++++ setup.go | 5 +++++ setup_bitswap.go | 1 + 5 files changed, 16 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index bb6995b..3b6df1d 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/felixge/httpsnoop v1.0.4 github.com/ipfs-shipyard/nopfs v0.0.12 github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231024163508-120e0c51ee3a - github.com/ipfs/boxo v0.23.0 + github.com/ipfs/boxo v0.23.1-0.20240920171645-1364a16755c3 github.com/ipfs/go-block-format v0.2.0 github.com/ipfs/go-cid v0.4.1 github.com/ipfs/go-datastore v0.6.0 diff --git a/go.sum b/go.sum index 467a624..e7baa1f 100644 --- a/go.sum +++ b/go.sum @@ -235,8 +235,8 @@ github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231024163508-120e0c51ee3a h1:MKG github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231024163508-120e0c51ee3a/go.mod h1:6EekK/jo+TynwSE/ZOiOJd4eEvRXoavEC3vquKtv4yI= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.23.0 h1:dY1PpcvPJ//VuUQ1TUd5TZvmaGuzxJ8dOP6mXaw+ke8= -github.com/ipfs/boxo v0.23.0/go.mod h1:ulu5I6avTmgGmvjuCaBRKwsaOOKjBfQw1EiOOQp8M6E= +github.com/ipfs/boxo v0.23.1-0.20240920171645-1364a16755c3 h1:STC1B6+L6toikFAHKCFvEheaWu9U+gPn1EuP3WnFfBw= +github.com/ipfs/boxo v0.23.1-0.20240920171645-1364a16755c3/go.mod h1:ulu5I6avTmgGmvjuCaBRKwsaOOKjBfQw1EiOOQp8M6E= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-bitswap v0.11.0 h1:j1WVvhDX1yhG32NTC9xfxnqycqYIlhzEzLXG/cU1HyQ= diff --git a/main.go b/main.go index b9c0e59..6db3cc8 100644 --- a/main.go +++ b/main.go @@ -260,6 +260,12 @@ Generate an identity seed and launch a gateway: EnvVars: []string{"RAINBOW_BITSWAP"}, Usage: "Controls if Bitswap is enabled (useful for testing or when remote backend is used instead)", }, + &cli.IntFlag{ + Name: "bitswap-wanthave-replace-size", + Value: 1024, + EnvVars: []string{"BITSWAP_WANTHAVE_REPLACE_SIZE"}, + Usage: "Replace WantHave requests with WantBlock for small blocks up to this size, zero to disable", + }, &cli.StringSliceFlag{ Name: "remote-backends", Value: cli.NewStringSlice(), @@ -439,6 +445,7 @@ share the same seed as long as the indexes are different. DHTRouting: dhtRouting, DHTSharedHost: cctx.Bool("dht-shared-host"), Bitswap: bitswap, + BitswapHaveReplaceSize: cctx.Int("bitswap-wanthave-replace-size"), IpnsMaxCacheTTL: cctx.Duration("ipns-max-cache-ttl"), DenylistSubs: cctx.StringSlice("denylists"), Peering: peeringAddrs, diff --git a/setup.go b/setup.go index 06ce132..ef6945b 100644 --- a/setup.go +++ b/setup.go @@ -107,6 +107,11 @@ type Config struct { IpnsMaxCacheTTL time.Duration Bitswap bool + // BitswapHaveReplaceSize tells the bitswap server to replace WantHave + // with WantBlock requests when the block size less then or equal to this + // value. Set to zero to disable replacement and avoid block size lookup. + BitswapHaveReplaceSize int + DenylistSubs []string Peering []peer.AddrInfo diff --git a/setup_bitswap.go b/setup_bitswap.go index 2f4046a..73fc0a3 100644 --- a/setup_bitswap.go +++ b/setup_bitswap.go @@ -60,6 +60,7 @@ func setupBitswapExchange(ctx context.Context, cfg Config, h host.Host, cr routi bitswap.ProvideEnabled(false), // When we don't have a block, don't reply. This reduces processment. bitswap.SetSendDontHaves(false), + bitswap.WithReplaceHasWithBlockMaxSize(cfg.BitswapHaveReplaceSize), ) bn.Start(bswap) return &noNotifyExchange{bswap} From c72297e0326a50b1977036b1c7bc6f34d482c3d6 Mon Sep 17 00:00:00 2001 From: gammazero <11790789+gammazero@users.noreply.github.com> Date: Fri, 20 Sep 2024 12:10:00 -0700 Subject: [PATCH 2/5] Update CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 160310e..641ddfd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ The following emojis are used to highlight certain changes: ## [Unreleased] ### Added +- Ability to specify the maximum blocksize that bitswap will replace WantHave with WantBlock responses, and to disable replacement when set to zero. ### Changed From 4c49dc1cb18d9dc2ca13d9343860e097b188ac82 Mon Sep 17 00:00:00 2001 From: gammazero <11790789+gammazero@users.noreply.github.com> Date: Fri, 20 Sep 2024 12:58:56 -0700 Subject: [PATCH 3/5] Consistent option name BitswapWantHaveReplaceSize --- go.mod | 2 +- go.sum | 4 +-- main.go | 64 ++++++++++++++++++++++++------------------------ setup.go | 9 ++++--- setup_bitswap.go | 2 +- 5 files changed, 41 insertions(+), 40 deletions(-) diff --git a/go.mod b/go.mod index 3b6df1d..cfe6d93 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/felixge/httpsnoop v1.0.4 github.com/ipfs-shipyard/nopfs v0.0.12 github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231024163508-120e0c51ee3a - github.com/ipfs/boxo v0.23.1-0.20240920171645-1364a16755c3 + github.com/ipfs/boxo v0.23.1-0.20240920193856-639ef65ebcb1 github.com/ipfs/go-block-format v0.2.0 github.com/ipfs/go-cid v0.4.1 github.com/ipfs/go-datastore v0.6.0 diff --git a/go.sum b/go.sum index e7baa1f..5feed64 100644 --- a/go.sum +++ b/go.sum @@ -235,8 +235,8 @@ github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231024163508-120e0c51ee3a h1:MKG github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231024163508-120e0c51ee3a/go.mod h1:6EekK/jo+TynwSE/ZOiOJd4eEvRXoavEC3vquKtv4yI= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.23.1-0.20240920171645-1364a16755c3 h1:STC1B6+L6toikFAHKCFvEheaWu9U+gPn1EuP3WnFfBw= -github.com/ipfs/boxo v0.23.1-0.20240920171645-1364a16755c3/go.mod h1:ulu5I6avTmgGmvjuCaBRKwsaOOKjBfQw1EiOOQp8M6E= +github.com/ipfs/boxo v0.23.1-0.20240920193856-639ef65ebcb1 h1:SRl2uyOMbLfo6uVeZ0k4sQt+dJGzyqsjE6GpqD5ouAY= +github.com/ipfs/boxo v0.23.1-0.20240920193856-639ef65ebcb1/go.mod h1:ulu5I6avTmgGmvjuCaBRKwsaOOKjBfQw1EiOOQp8M6E= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-bitswap v0.11.0 h1:j1WVvhDX1yhG32NTC9xfxnqycqYIlhzEzLXG/cU1HyQ= diff --git a/main.go b/main.go index 6db3cc8..f3d1f29 100644 --- a/main.go +++ b/main.go @@ -264,7 +264,7 @@ Generate an identity seed and launch a gateway: Name: "bitswap-wanthave-replace-size", Value: 1024, EnvVars: []string{"BITSWAP_WANTHAVE_REPLACE_SIZE"}, - Usage: "Replace WantHave requests with WantBlock for small blocks up to this size, zero to disable", + Usage: "Replace WantHave with WantBlock responses for small blocks up to this size, 0 to disable replacement", }, &cli.StringSliceFlag{ Name: "remote-backends", @@ -430,37 +430,37 @@ share the same seed as long as the indexes are different. } cfg := Config{ - DataDir: ddir, - BlockstoreType: cctx.String("blockstore"), - GatewayDomains: cctx.StringSlice("gateway-domains"), - SubdomainGatewayDomains: cctx.StringSlice("subdomain-gateway-domains"), - TrustlessGatewayDomains: cctx.StringSlice("trustless-gateway-domains"), - ConnMgrLow: cctx.Int("libp2p-connmgr-low"), - ConnMgrHi: cctx.Int("libp2p-connmgr-high"), - ConnMgrGrace: cctx.Duration("libp2p-connmgr-grace"), - MaxMemory: cctx.Uint64("libp2p-max-memory"), - MaxFD: cctx.Int("libp2p-max-fd"), - InMemBlockCache: cctx.Int64("inmem-block-cache"), - RoutingV1Endpoints: cctx.StringSlice("http-routers"), - DHTRouting: dhtRouting, - DHTSharedHost: cctx.Bool("dht-shared-host"), - Bitswap: bitswap, - BitswapHaveReplaceSize: cctx.Int("bitswap-wanthave-replace-size"), - IpnsMaxCacheTTL: cctx.Duration("ipns-max-cache-ttl"), - DenylistSubs: cctx.StringSlice("denylists"), - Peering: peeringAddrs, - PeeringSharedCache: cctx.Bool("peering-shared-cache"), - Seed: seed, - SeedIndex: index, - SeedPeering: seedPeering, - SeedPeeringMaxIndex: cctx.Int("seed-peering-max-index"), - RemoteBackends: remoteBackends, - RemoteBackendsIPNS: cctx.Bool("remote-backends-ipns"), - RemoteBackendMode: RemoteBackendMode(cctx.String("remote-backends-mode")), - GCInterval: cctx.Duration("gc-interval"), - GCThreshold: cctx.Float64("gc-threshold"), - ListenAddrs: cctx.StringSlice("libp2p-listen-addrs"), - TracingAuthToken: cctx.String("tracing-auth"), + DataDir: ddir, + BlockstoreType: cctx.String("blockstore"), + GatewayDomains: cctx.StringSlice("gateway-domains"), + SubdomainGatewayDomains: cctx.StringSlice("subdomain-gateway-domains"), + TrustlessGatewayDomains: cctx.StringSlice("trustless-gateway-domains"), + ConnMgrLow: cctx.Int("libp2p-connmgr-low"), + ConnMgrHi: cctx.Int("libp2p-connmgr-high"), + ConnMgrGrace: cctx.Duration("libp2p-connmgr-grace"), + MaxMemory: cctx.Uint64("libp2p-max-memory"), + MaxFD: cctx.Int("libp2p-max-fd"), + InMemBlockCache: cctx.Int64("inmem-block-cache"), + RoutingV1Endpoints: cctx.StringSlice("http-routers"), + DHTRouting: dhtRouting, + DHTSharedHost: cctx.Bool("dht-shared-host"), + Bitswap: bitswap, + BitswapWantHaveReplaceSize: cctx.Int("bitswap-wanthave-replace-size"), + IpnsMaxCacheTTL: cctx.Duration("ipns-max-cache-ttl"), + DenylistSubs: cctx.StringSlice("denylists"), + Peering: peeringAddrs, + PeeringSharedCache: cctx.Bool("peering-shared-cache"), + Seed: seed, + SeedIndex: index, + SeedPeering: seedPeering, + SeedPeeringMaxIndex: cctx.Int("seed-peering-max-index"), + RemoteBackends: remoteBackends, + RemoteBackendsIPNS: cctx.Bool("remote-backends-ipns"), + RemoteBackendMode: RemoteBackendMode(cctx.String("remote-backends-mode")), + GCInterval: cctx.Duration("gc-interval"), + GCThreshold: cctx.Float64("gc-threshold"), + ListenAddrs: cctx.StringSlice("libp2p-listen-addrs"), + TracingAuthToken: cctx.String("tracing-auth"), } var gnd *Node diff --git a/setup.go b/setup.go index ef6945b..fd18785 100644 --- a/setup.go +++ b/setup.go @@ -107,10 +107,11 @@ type Config struct { IpnsMaxCacheTTL time.Duration Bitswap bool - // BitswapHaveReplaceSize tells the bitswap server to replace WantHave - // with WantBlock requests when the block size less then or equal to this - // value. Set to zero to disable replacement and avoid block size lookup. - BitswapHaveReplaceSize int + // BitswapWantHaveReplaceSize tells the bitswap server to replace WantHave + // with WantBlock responses when the block size less then or equal to this + // value. Set to zero to disable replacement and avoid block size lookup + // when processing HaveWant requests. + BitswapWantHaveReplaceSize int DenylistSubs []string diff --git a/setup_bitswap.go b/setup_bitswap.go index 73fc0a3..3029506 100644 --- a/setup_bitswap.go +++ b/setup_bitswap.go @@ -60,7 +60,7 @@ func setupBitswapExchange(ctx context.Context, cfg Config, h host.Host, cr routi bitswap.ProvideEnabled(false), // When we don't have a block, don't reply. This reduces processment. bitswap.SetSendDontHaves(false), - bitswap.WithReplaceHasWithBlockMaxSize(cfg.BitswapHaveReplaceSize), + bitswap.WithWantHaveReplaceSize(cfg.BitswapWantHaveReplaceSize), ) bn.Start(bswap) return &noNotifyExchange{bswap} From fccbf5f1a4c8af93729858d4783b73bcc19dc0d2 Mon Sep 17 00:00:00 2001 From: gammazero <11790789+gammazero@users.noreply.github.com> Date: Sat, 28 Sep 2024 21:09:40 -0700 Subject: [PATCH 4/5] update boxo --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index cfe6d93..3a5560a 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/felixge/httpsnoop v1.0.4 github.com/ipfs-shipyard/nopfs v0.0.12 github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231024163508-120e0c51ee3a - github.com/ipfs/boxo v0.23.1-0.20240920193856-639ef65ebcb1 + github.com/ipfs/boxo v0.23.1-0.20240927234853-19a402b7dc34 github.com/ipfs/go-block-format v0.2.0 github.com/ipfs/go-cid v0.4.1 github.com/ipfs/go-datastore v0.6.0 diff --git a/go.sum b/go.sum index 5feed64..6a80d14 100644 --- a/go.sum +++ b/go.sum @@ -235,8 +235,8 @@ github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231024163508-120e0c51ee3a h1:MKG github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231024163508-120e0c51ee3a/go.mod h1:6EekK/jo+TynwSE/ZOiOJd4eEvRXoavEC3vquKtv4yI= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.23.1-0.20240920193856-639ef65ebcb1 h1:SRl2uyOMbLfo6uVeZ0k4sQt+dJGzyqsjE6GpqD5ouAY= -github.com/ipfs/boxo v0.23.1-0.20240920193856-639ef65ebcb1/go.mod h1:ulu5I6avTmgGmvjuCaBRKwsaOOKjBfQw1EiOOQp8M6E= +github.com/ipfs/boxo v0.23.1-0.20240927234853-19a402b7dc34 h1:/Etgc4IR0OUF+nIoNdqwu12EYuaSMpd7/Nc5wRLd67U= +github.com/ipfs/boxo v0.23.1-0.20240927234853-19a402b7dc34/go.mod h1:ulu5I6avTmgGmvjuCaBRKwsaOOKjBfQw1EiOOQp8M6E= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-bitswap v0.11.0 h1:j1WVvhDX1yhG32NTC9xfxnqycqYIlhzEzLXG/cU1HyQ= From c0d404e1882fb139c3dc93af115a72496723d604 Mon Sep 17 00:00:00 2001 From: Andrew Gillis <11790789+gammazero@users.noreply.github.com> Date: Mon, 30 Sep 2024 14:17:34 -0700 Subject: [PATCH 5/5] Update CHANGELOG.md Co-authored-by: Marcin Rataj --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 641ddfd..71271a1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,7 +14,7 @@ The following emojis are used to highlight certain changes: ## [Unreleased] ### Added -- Ability to specify the maximum blocksize that bitswap will replace WantHave with WantBlock responses, and to disable replacement when set to zero. +- Ability to specify the maximum blocksize that bitswap will replace WantHave with WantBlock responses, and to disable replacement when set to zero. [#165](https://github.com/ipfs/rainbow/pull/165) ### Changed