From 7330a686e229ef53e4071fe0098dda7674367443 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Wed, 3 Jul 2024 22:03:31 -0400 Subject: [PATCH] feat: add tests that don't make assumptions on the public network, and switch to running them --- .github/workflows/test.yml | 11 --- daemon.go | 13 +-- integration_test.go | 157 +++++++++++++++++++++++++++++++++++++ main.go | 75 +++++++++++------- test/e2e_test.go | 23 ++++++ test/tools.go | 10 ++- 6 files changed, 243 insertions(+), 46 deletions(-) create mode 100644 integration_test.go diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 280f328..38ec0ec 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -18,17 +18,6 @@ jobs: - uses: actions/checkout@v4 with: path: 'ipfs-check' - - name: Build and Start - run: | - cd ipfs-check - go build -o ipfs-check - ./ipfs-check & - sleep 300 - - uses: ipfs/download-ipfs-distribution-action@v1 - - name: Configure Kubo Gateway - run: | - ipfs init; - - uses: ipfs/start-ipfs-daemon-action@v1 - name: Tests working-directory: ipfs-check run: | diff --git a/daemon.go b/daemon.go index 4995a48..93c8540 100644 --- a/daemon.go +++ b/daemon.go @@ -31,9 +31,10 @@ type kademlia interface { } type daemon struct { - h host.Host - dht kademlia - dhtMessenger *dhtpb.ProtocolMessenger + h host.Host + dht kademlia + dhtMessenger *dhtpb.ProtocolMessenger + createTestHost func() (host.Host, error) } func newDaemon(ctx context.Context, acceleratedDHT bool) (*daemon, error) { @@ -85,7 +86,9 @@ func newDaemon(ctx context.Context, acceleratedDHT bool) (*daemon, error) { return nil, err } - return &daemon{h: h, dht: d, dhtMessenger: pm}, nil + return &daemon{h: h, dht: d, dhtMessenger: pm, createTestHost: func() (host.Host, error) { + return libp2p.New(libp2p.ConnectionGater(&privateAddrFilterConnectionGater{})) + }}, nil } func (d *daemon) mustStart() { @@ -151,7 +154,7 @@ func (d *daemon) runCheck(query url.Values) (*output, error) { } } - testHost, err := libp2p.New(libp2p.ConnectionGater(&privateAddrFilterConnectionGater{})) + testHost, err := d.createTestHost() if err != nil { return nil, fmt.Errorf("server error: %w", err) } diff --git a/integration_test.go b/integration_test.go new file mode 100644 index 0000000..517fe99 --- /dev/null +++ b/integration_test.go @@ -0,0 +1,157 @@ +package main + +import ( + "context" + "github.com/aschmahmann/ipfs-check/test" + bsnet "github.com/ipfs/boxo/bitswap/network" + bsserver "github.com/ipfs/boxo/bitswap/server" + "github.com/ipfs/boxo/blockstore" + blocks "github.com/ipfs/go-block-format" + "github.com/ipfs/go-cid" + "github.com/ipfs/go-datastore" + dssync "github.com/ipfs/go-datastore/sync" + "github.com/libp2p/go-libp2p" + dht "github.com/libp2p/go-libp2p-kad-dht" + mplex "github.com/libp2p/go-libp2p-mplex" + routinghelpers "github.com/libp2p/go-libp2p-routing-helpers" + "github.com/libp2p/go-libp2p/core/host" + "github.com/libp2p/go-libp2p/core/peer" + "github.com/libp2p/go-libp2p/core/protocol" + "github.com/libp2p/go-libp2p/p2p/net/connmgr" + "github.com/multiformats/go-multihash" + "github.com/stretchr/testify/require" + "testing" + "time" +) + +func TestBasicIntegration(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + testDHTPrefix := protocol.TestingID + testDHTID := protocol.TestingID + "/kad/1.0.0" + + dhtHost, err := libp2p.New() + require.NoError(t, err) + defer dhtHost.Close() + dhtServer, err := dht.New(ctx, dhtHost, dht.Mode(dht.ModeServer), dht.ProtocolPrefix(testDHTPrefix)) + require.NoError(t, err) + defer dhtServer.Close() + + go func() { + rm, err := NewResourceManager() + require.NoError(t, err) + + c, err := connmgr.NewConnManager(600, 900, connmgr.WithGracePeriod(time.Second*30)) + require.NoError(t, err) + + queryHost, err := libp2p.New( + libp2p.DefaultMuxers, + libp2p.Muxer(mplex.ID, mplex.DefaultTransport), + libp2p.ConnectionManager(c), + libp2p.ResourceManager(rm), + libp2p.EnableHolePunching(), + ) + require.NoError(t, err) + + pm, err := dhtProtocolMessenger(testDHTID, queryHost) + require.NoError(t, err) + queryDHT, err := dht.New(ctx, queryHost, dht.ProtocolPrefix(testDHTPrefix), dht.BootstrapPeers(peer.AddrInfo{ID: dhtHost.ID(), Addrs: dhtHost.Addrs()})) + require.NoError(t, err) + + d := &daemon{ + h: queryHost, + dht: queryDHT, + dhtMessenger: pm, + createTestHost: func() (host.Host, error) { + return libp2p.New(libp2p.DefaultMuxers, + libp2p.Muxer(mplex.ID, mplex.DefaultTransport), + libp2p.EnableHolePunching()) + }, + } + _ = startServer(ctx, d, ":1234") + }() + + h, err := libp2p.New() + defer h.Close() + require.NoError(t, err) + bn := bsnet.NewFromIpfsHost(h, routinghelpers.Null{}) + bstore := blockstore.NewBlockstore(dssync.MutexWrap(datastore.NewMapDatastore())) + bswap := bsserver.New(ctx, bn, bstore) + bn.Start(bswap) + defer bswap.Close() + dhtClient, err := dht.New(ctx, h, dht.ProtocolPrefix(testDHTPrefix), dht.Mode(dht.ModeClient), dht.BootstrapPeers(peer.AddrInfo{ID: dhtHost.ID(), Addrs: dhtHost.Addrs()})) + require.NoError(t, err) + defer dhtClient.Close() + err = dhtClient.Bootstrap(ctx) + require.NoError(t, err) + for dhtClient.RoutingTable().Size() == 0 { + select { + case <-ctx.Done(): + t.Fatal(ctx.Err()) + case <-time.After(time.Millisecond * 5): + } + } + + mas, err := peer.AddrInfoToP2pAddrs(&peer.AddrInfo{ID: h.ID(), Addrs: h.Addrs()}) + require.NoError(t, err) + hostAddr := mas[0] + + t.Run("Data on reachable peer that's advertised", func(t *testing.T) { + testData := []byte(t.Name()) + mh, err := multihash.Sum(testData, multihash.SHA2_256, -1) + require.NoError(t, err) + testCid := cid.NewCidV1(cid.Raw, mh) + testBlock, err := blocks.NewBlockWithCid(testData, testCid) + require.NoError(t, err) + err = bstore.Put(ctx, testBlock) + require.NoError(t, err) + err = dhtClient.Provide(ctx, testCid, true) + require.NoError(t, err) + + obj := test.Query(t, "http://localhost:1234", testCid.String(), hostAddr.String()) + + obj.Value("CidInDHT").Boolean().IsTrue() + obj.Value("ConnectionError").String().IsEmpty() + obj.Value("DataAvailableOverBitswap").Object().Value("Error").String().IsEmpty() + obj.Value("DataAvailableOverBitswap").Object().Value("Found").Boolean().IsTrue() + obj.Value("DataAvailableOverBitswap").Object().Value("Responded").Boolean().IsTrue() + }) + + t.Run("Data on reachable peer that's not advertised", func(t *testing.T) { + testData := []byte(t.Name()) + mh, err := multihash.Sum(testData, multihash.SHA2_256, -1) + require.NoError(t, err) + testCid := cid.NewCidV1(cid.Raw, mh) + testBlock, err := blocks.NewBlockWithCid(testData, testCid) + require.NoError(t, err) + err = bstore.Put(ctx, testBlock) + require.NoError(t, err) + + obj := test.Query(t, "http://localhost:1234", testCid.String(), hostAddr.String()) + + obj.Value("CidInDHT").Boolean().IsFalse() + obj.Value("ConnectionError").String().IsEmpty() + obj.Value("DataAvailableOverBitswap").Object().Value("Error").String().IsEmpty() + obj.Value("DataAvailableOverBitswap").Object().Value("Found").Boolean().IsTrue() + obj.Value("DataAvailableOverBitswap").Object().Value("Responded").Boolean().IsTrue() + }) + + t.Run("Data that's advertised but not served", func(t *testing.T) { + testData := []byte(t.Name()) + mh, err := multihash.Sum(testData, multihash.SHA2_256, -1) + require.NoError(t, err) + testCid := cid.NewCidV1(cid.Raw, mh) + require.NoError(t, err) + err = dhtClient.Provide(ctx, testCid, true) + require.NoError(t, err) + + obj := test.Query(t, "http://localhost:1234", testCid.String(), hostAddr.String()) + + obj.Value("CidInDHT").Boolean().IsTrue() + obj.Value("ConnectionError").String().IsEmpty() + obj.Value("DataAvailableOverBitswap").Object().Value("Error").String().IsEmpty() + obj.Value("DataAvailableOverBitswap").Object().Value("Found").Boolean().IsFalse() + obj.Value("DataAvailableOverBitswap").Object().Value("Responded").Boolean().IsTrue() + }) +} diff --git a/main.go b/main.go index feebfe8..785156b 100644 --- a/main.go +++ b/main.go @@ -1,6 +1,7 @@ package main import ( + "context" "encoding/json" "log" "net" @@ -28,44 +29,60 @@ func main() { Usage: "run the accelerated DHT client", }, } - app.Action = func(ctx *cli.Context) error { - daemon, err := newDaemon(ctx.Context, ctx.Bool("accelerated-dht")) + app.Action = func(cctx *cli.Context) error { + ctx := cctx.Context + d, err := newDaemon(ctx, cctx.Bool("accelerated-dht")) if err != nil { return err } + return startServer(ctx, d, cctx.String("address")) + } - l, err := net.Listen("tcp", ctx.String("address")) - if err != nil { - return err - } + err := app.Run(os.Args) + if err != nil { + log.Fatal(err) + } +} - log.Printf("listening on %v\n", l.Addr()) +func startServer(ctx context.Context, d *daemon, tcpListener string) error { + l, err := net.Listen("tcp", tcpListener) + if err != nil { + return err + } - daemon.mustStart() + log.Printf("listening on %v\n", l.Addr()) - log.Printf("ready to start serving") + d.mustStart() - // 1. Is the peer findable in the DHT? - // 2. Does the multiaddr work? If not, what's the error? - // 3. Is the CID in the DHT? - // 4. Does the peer respond that it has the given data over Bitswap? - http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { - w.Header().Add("Access-Control-Allow-Origin", "*") - data, err := daemon.runCheck(r.URL.Query()) - if err == nil { - w.Header().Add("Content-Type", "application/json") - _ = json.NewEncoder(w).Encode(data) - } else { - w.WriteHeader(http.StatusInternalServerError) - _, _ = w.Write([]byte(err.Error())) - } - }) + log.Printf("ready to start serving") - return http.Serve(l, nil) - } + // 1. Is the peer findable in the DHT? + // 2. Does the multiaddr work? If not, what's the error? + // 3. Is the CID in the DHT? + // 4. Does the peer respond that it has the given data over Bitswap? + http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + w.Header().Add("Access-Control-Allow-Origin", "*") + data, err := d.runCheck(r.URL.Query()) + if err == nil { + w.Header().Add("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(data) + } else { + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte(err.Error())) + } + }) - err := app.Run(os.Args) - if err != nil { - log.Fatal(err) + done := make(chan error, 1) + go func() { + defer close(done) + done <- http.Serve(l, nil) + }() + + select { + case err := <-done: + return err + case <-ctx.Done(): + _ = l.Close() + return <-done } } diff --git a/test/e2e_test.go b/test/e2e_test.go index 83209e8..a05a5cd 100644 --- a/test/e2e_test.go +++ b/test/e2e_test.go @@ -2,6 +2,7 @@ package test import ( "fmt" + "os" "testing" "time" ) @@ -15,9 +16,16 @@ var ( WIKIPEDIA_CID string WIKIPEDIA_PEER_ID string WIKIPEDIA_PEER_ADDR string + SKIP_E2E_TESTS bool ) func init() { + if os.Getenv("E2E_TESTS") == "true" { + SKIP_E2E_TESTS = false + } else { + SKIP_E2E_TESTS = true + return + } BOOTSTRAP_PEER_ADDR = call("bash", "-c", "ipfs bootstrap list | head -n 1") // ipfs name resolve /ipns/en.wikipedia-on-ipfs.org => /ipfs/CID, we remove the /ipfs/ prefix WIKIPEDIA_CID = call("ipfs", "name", "resolve", "/ipns/en.wikipedia-on-ipfs.org")[6:] @@ -26,6 +34,9 @@ func init() { } func TestEmptyDirOnBoostrapPeer(t *testing.T) { + if SKIP_E2E_TESTS { + t.Skip("Skipping e2e tests") + } obj := Q(t, EMPTY_DIR_CID, BOOTSTRAP_PEER_ADDR) obj.Value("CidInDHT").Boolean().IsTrue() @@ -36,6 +47,9 @@ func TestEmptyDirOnBoostrapPeer(t *testing.T) { } func TestWikipediaOnSomeProviderPeer(t *testing.T) { + if SKIP_E2E_TESTS { + t.Skip("Skipping e2e tests") + } obj := Q(t, WIKIPEDIA_CID, WIKIPEDIA_PEER_ADDR) obj.Value("CidInDHT").Boolean().IsTrue() // It seems that most peers do not provide over bitswap: @@ -46,6 +60,9 @@ func TestWikipediaOnSomeProviderPeer(t *testing.T) { } func TestRandomFileOnBootstrapPeer(t *testing.T) { + if SKIP_E2E_TESTS { + t.Skip("Skipping e2e tests") + } t.Skip("the random file CID is marked as \"not found in the DHT\" when calling bootstrap peers") randomFileCid := call("bash", "-c", "cat /dev/urandom | head | sha256sum | ipfs add --quiet -") @@ -65,6 +82,9 @@ func TestRandomFileOnBootstrapPeer(t *testing.T) { } func TestRandomFileOnLocalPeer(t *testing.T) { + if SKIP_E2E_TESTS { + t.Skip("Skipping e2e tests") + } // ipfs id -f "" nodeId := call("ipfs", "id", "-f", "") localAddr := fmt.Sprintf("/p2p/%s", nodeId) @@ -88,6 +108,9 @@ func TestRandomFileOnLocalPeer(t *testing.T) { } func TestRandomFileNeverUploadedOnBootstrapPeer(t *testing.T) { + if SKIP_E2E_TESTS { + t.Skip("Skipping e2e tests") + } randomFileCid := call("bash", "-c", "cat /dev/urandom | head | sha256sum | ipfs add --quiet --only-hash -") obj := Q(t, randomFileCid, BOOTSTRAP_PEER_ADDR) diff --git a/test/tools.go b/test/tools.go index 3e23bce..2e3552a 100644 --- a/test/tools.go +++ b/test/tools.go @@ -20,8 +20,16 @@ func Q( ) *httpexpect.Object { url := GetEnv("GATEWAY_URL", "http://localhost:3333") // url := GetEnv("GATEWAY_URL", "https://ipfs-check-backend.ipfs.io") - expectedContentType := "application/json" + return Query(t, url, cid, multiaddr) +} +func Query( + t *testing.T, + url string, + cid string, + multiaddr string, +) *httpexpect.Object { + expectedContentType := "application/json" if url == "https://ipfs-check-backend.ipfs.io" { // Temporary patch: the current released gateway returns text/plain. // TODO: when the correct Content-Type is released, remove all code related to this