fix: iron out error conditions, add test data debugging

pkg/internal/itest/client_retrieval_test.go

@@ -66,7 +66,7 @@ func TestRetrieval(t *testing.T) {
 			mrn.MN.LinkAll()
 
 			// Generate source data on the remote
-			srcData := tt.generate(t, mrn.Remotes[0].LinkSystem)
+			srcData := tt.generate(t, *mrn.Remotes[0].LinkSystem)
 
 			// Perform retrieval
 			linkSystemLocal := runRetrieval(t, ctx, mrn, srcData.Root, finishedChan)

pkg/internal/itest/direct_fetch_test.go

@@ -72,11 +72,11 @@ func TestDirectFetch(t *testing.T) {
 			// graphsync peer (0)
 			graphsyncMAs, err := peer.AddrInfoToP2pAddrs(mrn.Remotes[0].AddrInfo())
 			req.NoError(err)
-			srcData1 := unixfs.GenerateFile(t, &mrn.Remotes[0].LinkSystem, rndReader, 4<<20)
+			srcData1 := unixfs.GenerateFile(t, mrn.Remotes[0].LinkSystem, rndReader, 4<<20)
 			mocknet.SetupRetrieval(t, mrn.Remotes[0])
 
 			// bitswap peer (1)
-			srcData2 := unixfs.GenerateFile(t, &mrn.Remotes[1].LinkSystem, bytes.NewReader(srcData1.Content), 4<<20)
+			srcData2 := unixfs.GenerateFile(t, mrn.Remotes[1].LinkSystem, bytes.NewReader(srcData1.Content), 4<<20)
 			req.Equal(srcData2.Root, srcData2.Root)
 			bitswapMAs, err := peer.AddrInfoToP2pAddrs(mrn.Remotes[1].AddrInfo())
 			req.NoError(err)

pkg/internal/itest/testpeer/generator.go

@@ -129,7 +129,7 @@ type TestPeer struct {
 	blockstore         blockstore.Blockstore
 	Host               host.Host
 	blockstoreDelay    delay.D
-	LinkSystem         linking.LinkSystem
+	LinkSystem         *linking.LinkSystem
 	Cids               map[cid.Cid]struct{}
 }
 
@@ -182,7 +182,7 @@ func NewTestGraphsyncPeer(ctx context.Context, mn mocknet.Mocknet, p tnet.Identi
 	// Setup remote data transfer
 	gsNetRemote := gsnet.NewFromLibp2pHost(peer.Host)
 	dtNetRemote := dtnet.NewFromLibp2pHost(peer.Host, dtnet.RetryParameters(0, 0, 0, 0))
-	gsRemote := gsimpl.New(ctx, gsNetRemote, peer.LinkSystem)
+	gsRemote := gsimpl.New(ctx, gsNetRemote, *peer.LinkSystem)
 	gstpRemote := gstransport.NewTransport(peer.Host.ID(), gsRemote)
 	dtRemote, err := dtimpl.NewDataTransfer(dstore, dtNetRemote, gstpRemote)
 	if err != nil {
@@ -215,12 +215,13 @@ func newTestPeer(ctx context.Context, mn mocknet.Mocknet, p tnet.Identity) (Test
 	if err != nil {
 		return TestPeer{}, nil, err
 	}
+	lsys := storeutil.LinkSystemForBlockstore(bstore)
 	tp := TestPeer{
 		Host:            client,
 		ID:              p.ID(),
 		blockstore:      bstore,
 		blockstoreDelay: bsdelay,
-		LinkSystem:      storeutil.LinkSystemForBlockstore(bstore),
+		LinkSystem:      &lsys,
 		Cids:            make(map[cid.Cid]struct{}),
 	}
 

pkg/internal/itest/unixfs/directory.go

@@ -2,6 +2,8 @@ package unixfs
 
 import (
 	"context"
+	"crypto/sha256"
+	"encoding/hex"
 	"fmt"
 	"io"
 	"testing"
@@ -102,10 +104,12 @@ func toDirEntryRecursive(t *testing.T, linkSys linking.LinkSystem, rootCid cid.C
 }
 
 func CompareDirEntries(t *testing.T, a, b DirEntry) {
-	t.Log("CompareDirEntries", a.Path, b.Path) // TODO: remove this
+	// t.Log("CompareDirEntries", a.Path, b.Path) // TODO: remove this
 	require.Equal(t, a.Path, b.Path)
-	require.Equal(t, a.Content, b.Content, a.Path+" content mismatch")
-	require.Equal(t, a.Root, b.Root, a.Path+" root mismatch")
+	require.Equal(t, a.Root.String(), b.Root.String(), a.Path+" root mismatch")
+	hashA := sha256.Sum256(a.Content)
+	hashB := sha256.Sum256(b.Content)
+	require.Equal(t, hex.EncodeToString(hashA[:]), hex.EncodeToString(hashB[:]), a.Path+"content hash mismatch")
 	require.Equal(t, len(a.Children), len(b.Children), fmt.Sprintf("%s child length mismatch %d <> %d", a.Path, len(a.Children), len(b.Children)))
 	for i := range a.Children {
 		// not necessarily in order

pkg/internal/itest/unixfs/generator.go

@@ -5,6 +5,7 @@ import (
 	"crypto/rand"
 	"io"
 	"math/big"
+	"sort"
 	"strings"
 	"testing"
 
@@ -92,8 +93,15 @@ func GenerateDirectoryFrom(t *testing.T,
 			if targetSize-curSize <= 1024 { // don't make tiny directories
 				continue
 			}
-			newDir, err := namegen.RandomDirectoryName(randReader)
-			require.NoError(t, err)
+			var newDir string
+			for {
+				var err error
+				newDir, err = namegen.RandomDirectoryName(randReader)
+				require.NoError(t, err)
+				if !dupeName(children, newDir) {
+					break
+				}
+			}
 			child := GenerateDirectoryFrom(t, linkSys, randReader, targetSize-curSize, dir+"/"+newDir, false)
 			children = append(children, child)
 			curSize += int(child.TSize)
@@ -108,8 +116,15 @@ func GenerateDirectoryFrom(t *testing.T,
 				}
 			}
 			entry := GenerateFile(t, linkSys, randReader, size)
-			name, err := namegen.RandomFileName(randReader)
-			require.NoError(t, err)
+			var name string
+			for {
+				var err error
+				name, err = namegen.RandomFileName(randReader)
+				require.NoError(t, err)
+				if !dupeName(children, name) {
+					break
+				}
+			}
 			entry.Path = dir + "/" + name
 			curSize += size
 			children = append(children, entry)
@@ -120,7 +135,22 @@ func GenerateDirectoryFrom(t *testing.T,
 	return dirEntry
 }
 
+func dupeName(children []DirEntry, name string) bool {
+	for _, child := range children {
+		if strings.HasSuffix(child.Path, "/"+name) {
+			return true
+		}
+	}
+	return false
+}
+
 func BuildDirectory(t *testing.T, linkSys *linking.LinkSystem, children []DirEntry, sharded bool) DirEntry {
+	// create stable sorted children, which should match the encoded form
+	// in dag-pb
+	sort.Slice(children, func(i, j int) bool {
+		return strings.Compare(children[i].Path, children[j].Path) < 0
+	})
+
 	dirLinks := make([]dagpb.PBLink, 0)
 	for _, child := range children {
 		paths := strings.Split(child.Path, "/")

pkg/server/http/client_close_test.go

@@ -32,7 +32,7 @@ func TestHttpClientClose(t *testing.T) {
 	mrn.AddBitswapPeers(1)
 	require.NoError(t, mrn.MN.LinkAll())
 
-	srcData := unixfs.GenerateFile(t, &mrn.Remotes[0].LinkSystem, rndReader, 20<<20)
+	srcData := unixfs.GenerateFile(t, mrn.Remotes[0].LinkSystem, rndReader, 20<<20)
 
 	// Setup a new lassie
 	req := require.New(t)