From a10856b3e01e37a5b10a5179589a8a758394b7c4 Mon Sep 17 00:00:00 2001 From: Rod Vagg Date: Thu, 11 May 2023 22:08:41 +1000 Subject: [PATCH] fix: iron out error conditions, add test data debugging --- pkg/internal/itest/client_retrieval_test.go | 2 +- pkg/internal/itest/direct_fetch_test.go | 4 +- pkg/internal/itest/http_fetch_test.go | 155 ++++++++++++++------ pkg/internal/itest/testpeer/generator.go | 7 +- pkg/internal/itest/unixfs/directory.go | 10 +- pkg/internal/itest/unixfs/generator.go | 38 ++++- pkg/server/http/client_close_test.go | 2 +- 7 files changed, 158 insertions(+), 60 deletions(-) diff --git a/pkg/internal/itest/client_retrieval_test.go b/pkg/internal/itest/client_retrieval_test.go index 2e9edddd..97731880 100644 --- a/pkg/internal/itest/client_retrieval_test.go +++ b/pkg/internal/itest/client_retrieval_test.go @@ -66,7 +66,7 @@ func TestRetrieval(t *testing.T) { mrn.MN.LinkAll() // Generate source data on the remote - srcData := tt.generate(t, mrn.Remotes[0].LinkSystem) + srcData := tt.generate(t, *mrn.Remotes[0].LinkSystem) // Perform retrieval linkSystemLocal := runRetrieval(t, ctx, mrn, srcData.Root, finishedChan) diff --git a/pkg/internal/itest/direct_fetch_test.go b/pkg/internal/itest/direct_fetch_test.go index 7fda9436..972f91da 100644 --- a/pkg/internal/itest/direct_fetch_test.go +++ b/pkg/internal/itest/direct_fetch_test.go @@ -72,11 +72,11 @@ func TestDirectFetch(t *testing.T) { // graphsync peer (0) graphsyncMAs, err := peer.AddrInfoToP2pAddrs(mrn.Remotes[0].AddrInfo()) req.NoError(err) - srcData1 := unixfs.GenerateFile(t, &mrn.Remotes[0].LinkSystem, rndReader, 4<<20) + srcData1 := unixfs.GenerateFile(t, mrn.Remotes[0].LinkSystem, rndReader, 4<<20) mocknet.SetupRetrieval(t, mrn.Remotes[0]) // bitswap peer (1) - srcData2 := unixfs.GenerateFile(t, &mrn.Remotes[1].LinkSystem, bytes.NewReader(srcData1.Content), 4<<20) + srcData2 := unixfs.GenerateFile(t, mrn.Remotes[1].LinkSystem, bytes.NewReader(srcData1.Content), 4<<20) req.Equal(srcData2.Root, srcData2.Root) bitswapMAs, err := peer.AddrInfoToP2pAddrs(mrn.Remotes[1].AddrInfo()) req.NoError(err) diff --git a/pkg/internal/itest/http_fetch_test.go b/pkg/internal/itest/http_fetch_test.go index bd527bc7..c7935069 100644 --- a/pkg/internal/itest/http_fetch_test.go +++ b/pkg/internal/itest/http_fetch_test.go @@ -10,6 +10,7 @@ import ( "math/rand" "net/http" "net/url" + "os" "strings" "sync" "testing" @@ -23,7 +24,10 @@ import ( httpserver "github.com/filecoin-project/lassie/pkg/server/http" "github.com/ipfs/go-cid" carv2 "github.com/ipld/go-car/v2" + "github.com/ipld/go-car/v2/storage" "github.com/ipld/go-ipld-prime" + "github.com/ipld/go-ipld-prime/datamodel" + "github.com/ipld/go-ipld-prime/linking" cidlink "github.com/ipld/go-ipld-prime/linking/cid" "github.com/libp2p/go-libp2p/core/peer" "github.com/multiformats/go-multicodec" @@ -32,6 +36,11 @@ import ( _ "net/http/pprof" ) +// DEBUG_DATA, when true, will write source and received data to CARs +// for inspection if tests fail; otherwise they are cleaned up as tests +// proceed. +const DEBUG_DATA = true + func TestHttpFetch(t *testing.T) { fileQuery := func(q url.Values, _ []testpeer.TestPeer) { q.Set("car-scope", "file") @@ -59,42 +68,42 @@ func TestHttpFetch(t *testing.T) { name: "graphsync large sharded file", graphsyncRemotes: 1, generate: func(t *testing.T, rndReader io.Reader, remotes []testpeer.TestPeer) []unixfs.DirEntry { - return []unixfs.DirEntry{unixfs.GenerateFile(t, &remotes[0].LinkSystem, rndReader, 4<<20)} + return []unixfs.DirEntry{unixfs.GenerateFile(t, remotes[0].LinkSystem, rndReader, 4<<20)} }, }, { name: "bitswap large sharded file", bitswapRemotes: 1, generate: func(t *testing.T, rndReader io.Reader, remotes []testpeer.TestPeer) []unixfs.DirEntry { - return []unixfs.DirEntry{unixfs.GenerateFile(t, &remotes[0].LinkSystem, rndReader, 4<<20)} + return []unixfs.DirEntry{unixfs.GenerateFile(t, remotes[0].LinkSystem, rndReader, 4<<20)} }, }, { name: "graphsync large directory", graphsyncRemotes: 1, generate: func(t *testing.T, rndReader io.Reader, remotes []testpeer.TestPeer) []unixfs.DirEntry { - return []unixfs.DirEntry{unixfs.GenerateDirectory(t, &remotes[0].LinkSystem, rndReader, 16<<20, false)} + return []unixfs.DirEntry{unixfs.GenerateDirectory(t, remotes[0].LinkSystem, rndReader, 16<<20, false)} }, }, { name: "bitswap large directory", bitswapRemotes: 1, generate: func(t *testing.T, rndReader io.Reader, remotes []testpeer.TestPeer) []unixfs.DirEntry { - return []unixfs.DirEntry{unixfs.GenerateDirectory(t, &remotes[0].LinkSystem, rndReader, 16<<20, false)} + return []unixfs.DirEntry{unixfs.GenerateDirectory(t, remotes[0].LinkSystem, rndReader, 16<<20, false)} }, }, { name: "graphsync large sharded directory", graphsyncRemotes: 1, generate: func(t *testing.T, rndReader io.Reader, remotes []testpeer.TestPeer) []unixfs.DirEntry { - return []unixfs.DirEntry{unixfs.GenerateDirectory(t, &remotes[0].LinkSystem, rndReader, 16<<20, true)} + return []unixfs.DirEntry{unixfs.GenerateDirectory(t, remotes[0].LinkSystem, rndReader, 16<<20, true)} }, }, { name: "bitswap large sharded directory", bitswapRemotes: 1, generate: func(t *testing.T, rndReader io.Reader, remotes []testpeer.TestPeer) []unixfs.DirEntry { - return []unixfs.DirEntry{unixfs.GenerateDirectory(t, &remotes[0].LinkSystem, rndReader, 16<<20, true)} + return []unixfs.DirEntry{unixfs.GenerateDirectory(t, remotes[0].LinkSystem, rndReader, 16<<20, true)} }, }, { @@ -105,7 +114,7 @@ func TestHttpFetch(t *testing.T) { return cfg }, generate: func(t *testing.T, rndReader io.Reader, remotes []testpeer.TestPeer) []unixfs.DirEntry { - return []unixfs.DirEntry{unixfs.GenerateFile(t, &remotes[0].LinkSystem, rndReader, 4<<20)} + return []unixfs.DirEntry{unixfs.GenerateFile(t, remotes[0].LinkSystem, rndReader, 4<<20)} }, validateBodies: []bodyValidator{func(t *testing.T, srcData unixfs.DirEntry, body []byte) { // 3 blocks max, start at the root and then two blocks into the sharded data @@ -126,7 +135,7 @@ func TestHttpFetch(t *testing.T) { }, }, generate: func(t *testing.T, rndReader io.Reader, remotes []testpeer.TestPeer) []unixfs.DirEntry { - return []unixfs.DirEntry{unixfs.GenerateFile(t, &remotes[0].LinkSystem, rndReader, 4<<20)} + return []unixfs.DirEntry{unixfs.GenerateFile(t, remotes[0].LinkSystem, rndReader, 4<<20)} }, validateBodies: []bodyValidator{func(t *testing.T, srcData unixfs.DirEntry, body []byte) { // 3 blocks max, start at the root and then two blocks into the sharded data @@ -146,7 +155,7 @@ func TestHttpFetch(t *testing.T) { return cfg }, generate: func(t *testing.T, rndReader io.Reader, remotes []testpeer.TestPeer) []unixfs.DirEntry { - return []unixfs.DirEntry{unixfs.GenerateFile(t, &remotes[0].LinkSystem, rndReader, 4<<20)} + return []unixfs.DirEntry{unixfs.GenerateFile(t, remotes[0].LinkSystem, rndReader, 4<<20)} }, validateBodies: []bodyValidator{func(t *testing.T, srcData unixfs.DirEntry, body []byte) { // 3 blocks max, start at the root and then two blocks into the sharded data @@ -163,7 +172,7 @@ func TestHttpFetch(t *testing.T) { name: "graphsync large sharded file, car-scope file", graphsyncRemotes: 1, generate: func(t *testing.T, rndReader io.Reader, remotes []testpeer.TestPeer) []unixfs.DirEntry { - return []unixfs.DirEntry{unixfs.GenerateFile(t, &remotes[0].LinkSystem, rndReader, 4<<20)} + return []unixfs.DirEntry{unixfs.GenerateFile(t, remotes[0].LinkSystem, rndReader, 4<<20)} }, modifyQueries: []queryModifier{fileQuery}, }, @@ -172,7 +181,7 @@ func TestHttpFetch(t *testing.T) { name: "bitswap large sharded file, car-scope file", bitswapRemotes: 1, generate: func(t *testing.T, rndReader io.Reader, remotes []testpeer.TestPeer) []unixfs.DirEntry { - return []unixfs.DirEntry{unixfs.GenerateFile(t, &remotes[0].LinkSystem, rndReader, 4<<20)} + return []unixfs.DirEntry{unixfs.GenerateFile(t, remotes[0].LinkSystem, rndReader, 4<<20)} }, modifyQueries: []queryModifier{fileQuery}, }, @@ -180,7 +189,7 @@ func TestHttpFetch(t *testing.T) { name: "graphsync nested large sharded file, with path, car-scope file", graphsyncRemotes: 1, generate: func(t *testing.T, rndReader io.Reader, remotes []testpeer.TestPeer) []unixfs.DirEntry { - lsys := &remotes[0].LinkSystem + lsys := remotes[0].LinkSystem return []unixfs.DirEntry{wrapUnixfsContent(t, rndReader, lsys, unixfs.GenerateFile(t, lsys, rndReader, 4<<20))} }, paths: []string{"/want2/want1/want0"}, @@ -200,7 +209,7 @@ func TestHttpFetch(t *testing.T) { name: "bitswap nested large sharded file, with path, car-scope file", bitswapRemotes: 1, generate: func(t *testing.T, rndReader io.Reader, remotes []testpeer.TestPeer) []unixfs.DirEntry { - lsys := &remotes[0].LinkSystem + lsys := remotes[0].LinkSystem return []unixfs.DirEntry{wrapUnixfsContent(t, rndReader, lsys, unixfs.GenerateFile(t, lsys, rndReader, 4<<20))} }, paths: []string{"/want2/want1/want0"}, @@ -220,7 +229,7 @@ func TestHttpFetch(t *testing.T) { name: "graphsync large directory, car-scope file", graphsyncRemotes: 1, generate: func(t *testing.T, rndReader io.Reader, remotes []testpeer.TestPeer) []unixfs.DirEntry { - return []unixfs.DirEntry{unixfs.GenerateDirectory(t, &remotes[0].LinkSystem, rndReader, 16<<20, false)} + return []unixfs.DirEntry{unixfs.GenerateDirectory(t, remotes[0].LinkSystem, rndReader, 16<<20, false)} }, modifyQueries: []queryModifier{fileQuery}, validateBodies: []bodyValidator{func(t *testing.T, srcData unixfs.DirEntry, body []byte) { @@ -232,7 +241,7 @@ func TestHttpFetch(t *testing.T) { name: "bitswap large directory, car-scope file", bitswapRemotes: 1, generate: func(t *testing.T, rndReader io.Reader, remotes []testpeer.TestPeer) []unixfs.DirEntry { - return []unixfs.DirEntry{unixfs.GenerateDirectory(t, &remotes[0].LinkSystem, rndReader, 16<<20, false)} + return []unixfs.DirEntry{unixfs.GenerateDirectory(t, remotes[0].LinkSystem, rndReader, 16<<20, false)} }, modifyQueries: []queryModifier{fileQuery}, validateBodies: []bodyValidator{func(t *testing.T, srcData unixfs.DirEntry, body []byte) { @@ -244,8 +253,8 @@ func TestHttpFetch(t *testing.T) { name: "graphsync nested large directory, with path, car-scope file", graphsyncRemotes: 1, generate: func(t *testing.T, rndReader io.Reader, remotes []testpeer.TestPeer) []unixfs.DirEntry { - lsys := &remotes[0].LinkSystem - return []unixfs.DirEntry{wrapUnixfsContent(t, rndReader, lsys, unixfs.GenerateDirectory(t, &remotes[0].LinkSystem, rndReader, 16<<20, false))} + lsys := remotes[0].LinkSystem + return []unixfs.DirEntry{wrapUnixfsContent(t, rndReader, lsys, unixfs.GenerateDirectory(t, remotes[0].LinkSystem, rndReader, 16<<20, false))} }, paths: []string{"/want2/want1/want0"}, modifyQueries: []queryModifier{fileQuery}, @@ -264,8 +273,8 @@ func TestHttpFetch(t *testing.T) { name: "bitswap nested large directory, with path, car-scope file", bitswapRemotes: 1, generate: func(t *testing.T, rndReader io.Reader, remotes []testpeer.TestPeer) []unixfs.DirEntry { - lsys := &remotes[0].LinkSystem - return []unixfs.DirEntry{wrapUnixfsContent(t, rndReader, lsys, unixfs.GenerateDirectory(t, &remotes[0].LinkSystem, rndReader, 16<<20, false))} + lsys := remotes[0].LinkSystem + return []unixfs.DirEntry{wrapUnixfsContent(t, rndReader, lsys, unixfs.GenerateDirectory(t, remotes[0].LinkSystem, rndReader, 16<<20, false))} }, paths: []string{"/want2/want1/want0"}, modifyQueries: []queryModifier{fileQuery}, @@ -284,8 +293,8 @@ func TestHttpFetch(t *testing.T) { name: "graphsync nested large directory, with path, full", graphsyncRemotes: 1, generate: func(t *testing.T, rndReader io.Reader, remotes []testpeer.TestPeer) []unixfs.DirEntry { - lsys := &remotes[0].LinkSystem - return []unixfs.DirEntry{wrapUnixfsContent(t, rndReader, lsys, unixfs.GenerateDirectory(t, &remotes[0].LinkSystem, rndReader, 16<<20, false))} + lsys := remotes[0].LinkSystem + return []unixfs.DirEntry{wrapUnixfsContent(t, rndReader, lsys, unixfs.GenerateDirectory(t, remotes[0].LinkSystem, rndReader, 16<<20, false))} }, paths: []string{"/want2/want1/want0"}, validateBodies: []bodyValidator{func(t *testing.T, srcData unixfs.DirEntry, body []byte) { @@ -308,8 +317,8 @@ func TestHttpFetch(t *testing.T) { name: "bitswap nested large directory, with path, full", bitswapRemotes: 1, generate: func(t *testing.T, rndReader io.Reader, remotes []testpeer.TestPeer) []unixfs.DirEntry { - lsys := &remotes[0].LinkSystem - return []unixfs.DirEntry{wrapUnixfsContent(t, rndReader, lsys, unixfs.GenerateDirectory(t, &remotes[0].LinkSystem, rndReader, 16<<20, false))} + lsys := remotes[0].LinkSystem + return []unixfs.DirEntry{wrapUnixfsContent(t, rndReader, lsys, unixfs.GenerateDirectory(t, remotes[0].LinkSystem, rndReader, 16<<20, false))} }, paths: []string{"/want2/want1/want0"}, validateBodies: []bodyValidator{func(t *testing.T, srcData unixfs.DirEntry, body []byte) { @@ -332,7 +341,7 @@ func TestHttpFetch(t *testing.T) { name: "graphsync nested large sharded directory, car-scope file", graphsyncRemotes: 1, generate: func(t *testing.T, rndReader io.Reader, remotes []testpeer.TestPeer) []unixfs.DirEntry { - return []unixfs.DirEntry{unixfs.GenerateDirectory(t, &remotes[0].LinkSystem, rndReader, 16<<20, true)} + return []unixfs.DirEntry{unixfs.GenerateDirectory(t, remotes[0].LinkSystem, rndReader, 16<<20, true)} }, modifyQueries: []queryModifier{fileQuery}, validateBodies: []bodyValidator{func(t *testing.T, srcData unixfs.DirEntry, body []byte) { @@ -345,7 +354,7 @@ func TestHttpFetch(t *testing.T) { name: "bitswap nested large sharded directory, car-scope file", bitswapRemotes: 1, generate: func(t *testing.T, rndReader io.Reader, remotes []testpeer.TestPeer) []unixfs.DirEntry { - return []unixfs.DirEntry{unixfs.GenerateDirectory(t, &remotes[0].LinkSystem, rndReader, 16<<20, true)} + return []unixfs.DirEntry{unixfs.GenerateDirectory(t, remotes[0].LinkSystem, rndReader, 16<<20, true)} }, modifyQueries: []queryModifier{fileQuery}, validateBodies: []bodyValidator{func(t *testing.T, srcData unixfs.DirEntry, body []byte) { @@ -358,8 +367,8 @@ func TestHttpFetch(t *testing.T) { name: "graphsync nested large sharded directory, with path, car-scope file", graphsyncRemotes: 1, generate: func(t *testing.T, rndReader io.Reader, remotes []testpeer.TestPeer) []unixfs.DirEntry { - lsys := &remotes[0].LinkSystem - return []unixfs.DirEntry{wrapUnixfsContent(t, rndReader, lsys, unixfs.GenerateDirectory(t, &remotes[0].LinkSystem, rndReader, 16<<20, true))} + lsys := remotes[0].LinkSystem + return []unixfs.DirEntry{wrapUnixfsContent(t, rndReader, lsys, unixfs.GenerateDirectory(t, remotes[0].LinkSystem, rndReader, 16<<20, true))} }, paths: []string{"/want2/want1/want0"}, modifyQueries: []queryModifier{fileQuery}, @@ -378,8 +387,8 @@ func TestHttpFetch(t *testing.T) { name: "bitswap nested large sharded directory, with path, car-scope file", bitswapRemotes: 1, generate: func(t *testing.T, rndReader io.Reader, remotes []testpeer.TestPeer) []unixfs.DirEntry { - lsys := &remotes[0].LinkSystem - return []unixfs.DirEntry{wrapUnixfsContent(t, rndReader, lsys, unixfs.GenerateDirectory(t, &remotes[0].LinkSystem, rndReader, 16<<20, true))} + lsys := remotes[0].LinkSystem + return []unixfs.DirEntry{wrapUnixfsContent(t, rndReader, lsys, unixfs.GenerateDirectory(t, remotes[0].LinkSystem, rndReader, 16<<20, true))} }, paths: []string{"/want2/want1/want0"}, modifyQueries: []queryModifier{fileQuery}, @@ -398,8 +407,8 @@ func TestHttpFetch(t *testing.T) { name: "graphsync nested large sharded directory, with path, full", graphsyncRemotes: 1, generate: func(t *testing.T, rndReader io.Reader, remotes []testpeer.TestPeer) []unixfs.DirEntry { - lsys := &remotes[0].LinkSystem - return []unixfs.DirEntry{wrapUnixfsContent(t, rndReader, lsys, unixfs.GenerateDirectory(t, &remotes[0].LinkSystem, rndReader, 16<<20, true))} + lsys := remotes[0].LinkSystem + return []unixfs.DirEntry{wrapUnixfsContent(t, rndReader, lsys, unixfs.GenerateDirectory(t, remotes[0].LinkSystem, rndReader, 16<<20, true))} }, paths: []string{"/want2/want1/want0"}, validateBodies: []bodyValidator{func(t *testing.T, srcData unixfs.DirEntry, body []byte) { @@ -422,8 +431,8 @@ func TestHttpFetch(t *testing.T) { name: "bitswap nested large sharded directory, with path, full", bitswapRemotes: 1, generate: func(t *testing.T, rndReader io.Reader, remotes []testpeer.TestPeer) []unixfs.DirEntry { - lsys := &remotes[0].LinkSystem - return []unixfs.DirEntry{wrapUnixfsContent(t, rndReader, lsys, unixfs.GenerateDirectory(t, &remotes[0].LinkSystem, rndReader, 16<<20, true))} + lsys := remotes[0].LinkSystem + return []unixfs.DirEntry{wrapUnixfsContent(t, rndReader, lsys, unixfs.GenerateDirectory(t, remotes[0].LinkSystem, rndReader, 16<<20, true))} }, paths: []string{"/want2/want1/want0"}, validateBodies: []bodyValidator{func(t *testing.T, srcData unixfs.DirEntry, body []byte) { @@ -498,8 +507,8 @@ func TestHttpFetch(t *testing.T) { bitswapRemotes: 2, generate: func(t *testing.T, rndReader io.Reader, remotes []testpeer.TestPeer) []unixfs.DirEntry { return []unixfs.DirEntry{ - unixfs.GenerateFile(t, &remotes[0].LinkSystem, rndReader, 4<<20), - unixfs.GenerateDirectory(t, &remotes[1].LinkSystem, rndReader, 16<<20, false), + unixfs.GenerateFile(t, remotes[0].LinkSystem, rndReader, 4<<20), + unixfs.GenerateDirectory(t, remotes[1].LinkSystem, rndReader, 16<<20, false), } }, }, @@ -508,8 +517,8 @@ func TestHttpFetch(t *testing.T) { graphsyncRemotes: 2, generate: func(t *testing.T, rndReader io.Reader, remotes []testpeer.TestPeer) []unixfs.DirEntry { return []unixfs.DirEntry{ - unixfs.GenerateFile(t, &remotes[0].LinkSystem, rndReader, 4<<20), - unixfs.GenerateDirectory(t, &remotes[1].LinkSystem, rndReader, 16<<20, false), + unixfs.GenerateFile(t, remotes[0].LinkSystem, rndReader, 4<<20), + unixfs.GenerateDirectory(t, remotes[1].LinkSystem, rndReader, 16<<20, false), } }, }, @@ -520,8 +529,8 @@ func TestHttpFetch(t *testing.T) { expectFail: true, generate: func(t *testing.T, rndReader io.Reader, remotes []testpeer.TestPeer) []unixfs.DirEntry { return []unixfs.DirEntry{ - unixfs.GenerateFile(t, &remotes[0].LinkSystem, rndReader, 4<<20), - unixfs.GenerateDirectory(t, &remotes[1].LinkSystem, rndReader, 16<<20, false), + unixfs.GenerateFile(t, remotes[0].LinkSystem, rndReader, 4<<20), + unixfs.GenerateDirectory(t, remotes[1].LinkSystem, rndReader, 16<<20, false), } }, }, @@ -531,8 +540,8 @@ func TestHttpFetch(t *testing.T) { bitswapRemotes: 1, generate: func(t *testing.T, rndReader io.Reader, remotes []testpeer.TestPeer) []unixfs.DirEntry { return []unixfs.DirEntry{ - unixfs.GenerateFile(t, &remotes[0].LinkSystem, rndReader, 4<<20), - unixfs.GenerateDirectory(t, &remotes[1].LinkSystem, rndReader, 16<<20, false), + unixfs.GenerateFile(t, remotes[0].LinkSystem, rndReader, 4<<20), + unixfs.GenerateDirectory(t, remotes[1].LinkSystem, rndReader, 16<<20, false), } }, }, @@ -541,7 +550,7 @@ func TestHttpFetch(t *testing.T) { name: "graphsync large sharded file, car-scope block", graphsyncRemotes: 1, generate: func(t *testing.T, rndReader io.Reader, remotes []testpeer.TestPeer) []unixfs.DirEntry { - return []unixfs.DirEntry{unixfs.GenerateFile(t, &remotes[0].LinkSystem, rndReader, 4<<20)} + return []unixfs.DirEntry{unixfs.GenerateFile(t, remotes[0].LinkSystem, rndReader, 4<<20)} }, modifyQueries: []queryModifier{blockQuery}, validateBodies: []bodyValidator{ @@ -557,7 +566,7 @@ func TestHttpFetch(t *testing.T) { name: "graphsync nested large sharded file, with path, car-scope block", graphsyncRemotes: 1, generate: func(t *testing.T, rndReader io.Reader, remotes []testpeer.TestPeer) []unixfs.DirEntry { - lsys := &remotes[0].LinkSystem + lsys := remotes[0].LinkSystem return []unixfs.DirEntry{wrapUnixfsContent(t, rndReader, lsys, unixfs.GenerateFile(t, lsys, rndReader, 4<<20))} }, paths: []string{"/want2/want1/want0"}, @@ -576,7 +585,7 @@ func TestHttpFetch(t *testing.T) { name: "graphsync large sharded file, fixedPeer", graphsyncRemotes: 1, generate: func(t *testing.T, rndReader io.Reader, remotes []testpeer.TestPeer) []unixfs.DirEntry { - fileEntry := unixfs.GenerateFile(t, &remotes[0].LinkSystem, rndReader, 4<<20) + fileEntry := unixfs.GenerateFile(t, remotes[0].LinkSystem, rndReader, 4<<20) // wipe content routing information for remote remotes[0].Cids = make(map[cid.Cid]struct{}) return []unixfs.DirEntry{fileEntry} @@ -594,7 +603,7 @@ func TestHttpFetch(t *testing.T) { name: "bitswap large sharded file", bitswapRemotes: 1, generate: func(t *testing.T, rndReader io.Reader, remotes []testpeer.TestPeer) []unixfs.DirEntry { - fileEntry := unixfs.GenerateFile(t, &remotes[0].LinkSystem, rndReader, 4<<20) + fileEntry := unixfs.GenerateFile(t, remotes[0].LinkSystem, rndReader, 4<<20) // wipe content routing information for remote remotes[0].Cids = make(map[cid.Cid]struct{}) return []unixfs.DirEntry{fileEntry} @@ -629,6 +638,7 @@ func TestHttpFetch(t *testing.T) { mrn.AddBitswapPeers(testCase.bitswapRemotes) require.NoError(t, mrn.MN.LinkAll()) + carFiles := debugRemotes(t, ctx, testCase.name, mrn.Remotes) srcData := testCase.generate(t, rndReader, mrn.Remotes) // Setup a new lassie @@ -717,15 +727,34 @@ func TestHttpFetch(t *testing.T) { err = resp.Body.Close() req.NoError(err) + if DEBUG_DATA { + dstf, err := os.CreateTemp("", fmt.Sprintf("%s_received%d.car", testCase.name, i)) + req.NoError(err) + t.Logf("Writing received data to CAR @ %s", dstf.Name()) + _, err = dstf.Write(body) + req.NoError(err) + carFiles = append(carFiles, dstf) + } + if testCase.validateBodies != nil && testCase.validateBodies[i] != nil { testCase.validateBodies[i](t, srcData[i], body) } else { - gotDir := unixfs.CarToDirEntry(t, bytes.NewReader(body), srcData[i].Root, true) + // gotDir := unixfs.CarToDirEntry(t, bytes.NewReader(body), srcData[i].Root, true) + gotLsys := unixfs.CarBytesLinkSystem(t, bytes.NewReader(body)) + gotDir := unixfs.ToDirEntry(t, gotLsys, srcData[i].Root, true) unixfs.CompareDirEntries(t, srcData[i], gotDir) } } } + if DEBUG_DATA { + for _, cf := range carFiles { + req.NoError(cf.Close()) + req.NoError(os.Remove(cf.Name())) + } + t.Logf("Cleaned up CARs") + } + err = httpServer.Close() req.NoError(err) select { @@ -798,3 +827,37 @@ func wrapUnixfsContent(t *testing.T, rndReader io.Reader, lsys *ipld.LinkSystem, return want } + +func debugRemotes(t *testing.T, ctx context.Context, name string, remotes []testpeer.TestPeer) []*os.File { + if !DEBUG_DATA { + return nil + } + carFiles := make([]*os.File, 0) + for ii, r := range remotes { + func(ii int, r testpeer.TestPeer) { + carFile, err := os.CreateTemp("", fmt.Sprintf("%s_remote%d.car", name, ii)) + require.NoError(t, err) + t.Logf("Writing source data to CAR @ %s", carFile.Name()) + carFiles = append(carFiles, carFile) + carW, err := storage.NewWritable(carFile, []cid.Cid{}, carv2.WriteAsCarV1(true), carv2.AllowDuplicatePuts(true)) + require.NoError(t, err) + swo := r.LinkSystem.StorageWriteOpener + r.LinkSystem.StorageWriteOpener = func(lc linking.LinkContext) (io.Writer, linking.BlockWriteCommitter, error) { + w, c, err := swo(lc) + if err != nil { + return nil, nil, err + } + var buf bytes.Buffer + return &buf, func(l datamodel.Link) error { + require.NoError(t, carW.Put(ctx, l.(cidlink.Link).Cid.KeyString(), buf.Bytes())) + _, err := w.Write(buf.Bytes()) + if err != nil { + return err + } + return c(l) + }, nil + } + }(ii, r) + } + return carFiles +} diff --git a/pkg/internal/itest/testpeer/generator.go b/pkg/internal/itest/testpeer/generator.go index 056f9e83..03863d72 100644 --- a/pkg/internal/itest/testpeer/generator.go +++ b/pkg/internal/itest/testpeer/generator.go @@ -129,7 +129,7 @@ type TestPeer struct { blockstore blockstore.Blockstore Host host.Host blockstoreDelay delay.D - LinkSystem linking.LinkSystem + LinkSystem *linking.LinkSystem Cids map[cid.Cid]struct{} } @@ -182,7 +182,7 @@ func NewTestGraphsyncPeer(ctx context.Context, mn mocknet.Mocknet, p tnet.Identi // Setup remote data transfer gsNetRemote := gsnet.NewFromLibp2pHost(peer.Host) dtNetRemote := dtnet.NewFromLibp2pHost(peer.Host, dtnet.RetryParameters(0, 0, 0, 0)) - gsRemote := gsimpl.New(ctx, gsNetRemote, peer.LinkSystem) + gsRemote := gsimpl.New(ctx, gsNetRemote, *peer.LinkSystem) gstpRemote := gstransport.NewTransport(peer.Host.ID(), gsRemote) dtRemote, err := dtimpl.NewDataTransfer(dstore, dtNetRemote, gstpRemote) if err != nil { @@ -215,12 +215,13 @@ func newTestPeer(ctx context.Context, mn mocknet.Mocknet, p tnet.Identity) (Test if err != nil { return TestPeer{}, nil, err } + lsys := storeutil.LinkSystemForBlockstore(bstore) tp := TestPeer{ Host: client, ID: p.ID(), blockstore: bstore, blockstoreDelay: bsdelay, - LinkSystem: storeutil.LinkSystemForBlockstore(bstore), + LinkSystem: &lsys, Cids: make(map[cid.Cid]struct{}), } diff --git a/pkg/internal/itest/unixfs/directory.go b/pkg/internal/itest/unixfs/directory.go index 5f80179e..08566b21 100644 --- a/pkg/internal/itest/unixfs/directory.go +++ b/pkg/internal/itest/unixfs/directory.go @@ -2,6 +2,8 @@ package unixfs import ( "context" + "crypto/sha256" + "encoding/hex" "fmt" "io" "testing" @@ -102,10 +104,12 @@ func toDirEntryRecursive(t *testing.T, linkSys linking.LinkSystem, rootCid cid.C } func CompareDirEntries(t *testing.T, a, b DirEntry) { - t.Log("CompareDirEntries", a.Path, b.Path) // TODO: remove this + // t.Log("CompareDirEntries", a.Path, b.Path) // TODO: remove this require.Equal(t, a.Path, b.Path) - require.Equal(t, a.Content, b.Content, a.Path+" content mismatch") - require.Equal(t, a.Root, b.Root, a.Path+" root mismatch") + require.Equal(t, a.Root.String(), b.Root.String(), a.Path+" root mismatch") + hashA := sha256.Sum256(a.Content) + hashB := sha256.Sum256(b.Content) + require.Equal(t, hex.EncodeToString(hashA[:]), hex.EncodeToString(hashB[:]), a.Path+"content hash mismatch") require.Equal(t, len(a.Children), len(b.Children), fmt.Sprintf("%s child length mismatch %d <> %d", a.Path, len(a.Children), len(b.Children))) for i := range a.Children { // not necessarily in order diff --git a/pkg/internal/itest/unixfs/generator.go b/pkg/internal/itest/unixfs/generator.go index 9a25b41d..00355e25 100644 --- a/pkg/internal/itest/unixfs/generator.go +++ b/pkg/internal/itest/unixfs/generator.go @@ -5,6 +5,7 @@ import ( "crypto/rand" "io" "math/big" + "sort" "strings" "testing" @@ -92,8 +93,15 @@ func GenerateDirectoryFrom(t *testing.T, if targetSize-curSize <= 1024 { // don't make tiny directories continue } - newDir, err := namegen.RandomDirectoryName(randReader) - require.NoError(t, err) + var newDir string + for { + var err error + newDir, err = namegen.RandomDirectoryName(randReader) + require.NoError(t, err) + if !dupeName(children, newDir) { + break + } + } child := GenerateDirectoryFrom(t, linkSys, randReader, targetSize-curSize, dir+"/"+newDir, false) children = append(children, child) curSize += int(child.TSize) @@ -108,8 +116,15 @@ func GenerateDirectoryFrom(t *testing.T, } } entry := GenerateFile(t, linkSys, randReader, size) - name, err := namegen.RandomFileName(randReader) - require.NoError(t, err) + var name string + for { + var err error + name, err = namegen.RandomFileName(randReader) + require.NoError(t, err) + if !dupeName(children, name) { + break + } + } entry.Path = dir + "/" + name curSize += size children = append(children, entry) @@ -120,7 +135,22 @@ func GenerateDirectoryFrom(t *testing.T, return dirEntry } +func dupeName(children []DirEntry, name string) bool { + for _, child := range children { + if strings.HasSuffix(child.Path, "/"+name) { + return true + } + } + return false +} + func BuildDirectory(t *testing.T, linkSys *linking.LinkSystem, children []DirEntry, sharded bool) DirEntry { + // create stable sorted children, which should match the encoded form + // in dag-pb + sort.Slice(children, func(i, j int) bool { + return strings.Compare(children[i].Path, children[j].Path) < 0 + }) + dirLinks := make([]dagpb.PBLink, 0) for _, child := range children { paths := strings.Split(child.Path, "/") diff --git a/pkg/server/http/client_close_test.go b/pkg/server/http/client_close_test.go index e28ff65b..4323fb74 100644 --- a/pkg/server/http/client_close_test.go +++ b/pkg/server/http/client_close_test.go @@ -32,7 +32,7 @@ func TestHttpClientClose(t *testing.T) { mrn.AddBitswapPeers(1) require.NoError(t, mrn.MN.LinkAll()) - srcData := unixfs.GenerateFile(t, &mrn.Remotes[0].LinkSystem, rndReader, 20<<20) + srcData := unixfs.GenerateFile(t, mrn.Remotes[0].LinkSystem, rndReader, 20<<20) // Setup a new lassie req := require.New(t)