fix(backend): fix SQLite JSON blob bugs

backend/hyper/entity.go

@@ -13,6 +13,7 @@ import (
 	"mintter/backend/hyper/hypersql"
 	"mintter/backend/ipfs"
 	"mintter/backend/pkg/dqb"
+	"mintter/backend/pkg/strbytes"
 	"sort"
 	"strings"
 
@@ -682,7 +683,7 @@ func (bs *Storage) LoadEntityFromHeads(ctx context.Context, eid EntityID, heads
 
 	defer sqlitex.Save(conn)(&err)
 
-	localHeads := make([]int64, 0, len(heads))
+	dbheads := make([]int64, 0, len(heads))
 	for _, c := range heads {
 		res, err := hypersql.BlobsGetSize(conn, c.Hash())
 		if err != nil {
@@ -691,26 +692,30 @@ func (bs *Storage) LoadEntityFromHeads(ctx context.Context, eid EntityID, heads
 		if res.BlobsID == 0 || res.BlobsSize < 0 {
 			return nil, status.Errorf(codes.NotFound, "no such head %s for entity %s", c, eid)
 		}
-		localHeads = append(localHeads, res.BlobsID)
+		dbheads = append(dbheads, res.BlobsID)
 	}
 
-	if len(localHeads) != len(heads) {
+	if len(dbheads) != len(heads) {
 		return nil, fmt.Errorf("couldn't resolve all the heads %v for entity %s", heads, eid)
 	}
 
-	jsonheads, err := json.Marshal(localHeads)
+	jsonheads, err := json.Marshal(dbheads)
 	if err != nil {
 		return nil, err
 	}
 
-	return bs.loadFromHeads(conn, eid, jsonheads)
+	return bs.loadFromHeads(conn, eid, localHeads(strbytes.String(jsonheads)))
 }
 
 // localHeads is a JSON-encoded array of integers corresponding to heads.
-type localHeads []byte
+type localHeads string
 
 func (bs *Storage) loadFromHeads(conn *sqlite.Conn, eid EntityID, heads localHeads) (e *Entity, err error) {
-	cset, err := hypersql.ChangesResolveHeads(conn, heads)
+	if heads == "" || heads == "null" {
+		heads = "[]"
+	}
+
+	cset, err := hypersql.ChangesResolveHeads(conn, string(heads))
 	if err != nil {
 		return nil, err
 	}

backend/hyper/hypersql/queries.gensum

@@ -1,2 +1,2 @@
-srcs: 2e1bd3b9f4e7baaf51f43553662842dc
-outs: 63faeb2bdfe750e843383133579a0550
+srcs: e80150f42251938329daac92fb5a9c42
+outs: ff29159baa5946b69e98f85b93b400a4

backend/hyper/hypersql/queries.go

@@ -221,7 +221,7 @@ func generateQueries() error {
 				s.StructuralBlobsViewMultihash,
 				s.StructuralBlobsViewSize,
 			), '\n',
-			"FROM", qb.Concat(s.StructuralBlobsView, ", ", "json_each(", qb.Var("cset", sgen.TypeBytes), ") AS cset"), '\n',
+			"FROM", qb.Concat(s.StructuralBlobsView, ", ", "json_each(", qb.Var("cset", sgen.TypeText), ") AS cset"), '\n',
 			"WHERE", s.StructuralBlobsViewResource, "=", qb.VarColType(s.StructuralBlobsViewResource, sgen.TypeText), '\n',
 			"AND", s.StructuralBlobsViewBlobID, "= cset.value", '\n',
 			"ORDER BY", s.StructuralBlobsViewTs,
@@ -245,14 +245,14 @@ func generateQueries() error {
 		qb.MakeQuery(s.Schema, "ChangesResolveHeads", sgen.QueryKindSingle,
 			"WITH RECURSIVE changeset (change) AS", qb.SubQuery(
 				"SELECT value",
-				"FROM", qb.Concat("json_each(", qb.Var("heads", sgen.TypeBytes), ")"),
+				"FROM", qb.Concat("json_each(", qb.Var("heads", sgen.TypeText), ")"),
 				"UNION",
 				"SELECT", storage.ChangeDepsParent,
 				"FROM", storage.ChangeDeps,
 				"JOIN changeset ON changeset.change", "=", storage.ChangeDepsChild,
 			), '\n',
 			"SELECT", qb.Results(
-				qb.ResultRaw("json_group_array(change) AS resolved_json", "resolved_json", sgen.TypeBytes),
+				qb.ResultRaw("json_group_array(change) AS resolved_json", "resolved_json", sgen.TypeText),
 			), '\n',
 			"FROM changeset", '\n',
 			"LIMIT 1",
@@ -264,7 +264,7 @@ func generateQueries() error {
 				{Name: "entity", Type: sgen.TypeInt},
 			},
 			Outputs: []sgen.GoSymbol{
-				{Name: "Heads", Type: sgen.TypeBytes},
+				{Name: "Heads", Type: sgen.TypeText},
 			},
 			SQL: `WITH
 non_drafts (blob) AS (

backend/pkg/strbytes/strbytes.go

@@ -0,0 +1,23 @@
+// Package strbytes provides convenience wrappers for the *unsafe* []byte <-> string conversions.
+// No []byte value must be modified after it has been converted to or from a string.
+package strbytes
+
+import "unsafe"
+
+// String from bytes.
+func String(b []byte) string {
+	if len(b) == 0 {
+		return ""
+	}
+
+	return unsafe.String(&b[0], len(b))
+}
+
+// Bytes from string.
+func Bytes(s string) []byte {
+	if len(s) == 0 {
+		return nil
+	}
+
+	return unsafe.Slice(unsafe.StringData(s), len(s))
+}

backend/pkg/strbytes/strbytes_test.go

@@ -0,0 +1,34 @@
+package strbytes
+
+import (
+	"bytes"
+	"testing"
+)
+
+func TestAll(t *testing.T) {
+	roundTripString(t, "Hello")
+	roundTripString(t, "Привет! Как дела?")
+	roundTripString(t, "Hello, 世界")
+	roundTripString(t, "Hello, 👨‍👩‍👧‍👦")
+	roundTripBytes(t, []byte{143, 11, 254, 254, 168})
+}
+
+func roundTripString(t *testing.T, s string) {
+	b := Bytes(s)
+	s2 := String(b)
+	if s != s2 {
+		t.Fatalf("expected %q, got %q", s, s2)
+	}
+}
+
+func roundTripBytes(t *testing.T, b []byte) {
+	s := String(b)
+	b2 := Bytes(s)
+	if string(b) != string(b2) {
+		t.Fatalf("expected %q, got %q", b, b2)
+	}
+
+	if !bytes.Equal(b, b2) {
+		t.Fatalf("expected %q, got %q", b, b2)
+	}
+}