feat(doc-storage): impl doc storages

.eslintrc.js

@@ -54,7 +54,7 @@ const allPackages = [
   'packages/common/debug',
   'packages/common/env',
   'packages/common/infra',
-  'packages/common/theme',
+  'packages/common/doc-storage',
   'tools/cli',
 ];
 

Cargo.toml

@@ -19,6 +19,7 @@ serde        = "1"
 serde_json   = "1"
 sha3         = "0.10"
 sqlx         = { version = "0.8", default-features = false, features = ["chrono", "macros", "migrate", "runtime-tokio", "sqlite", "tls-rustls"] }
+log          = "0.4"
 tiktoken-rs  = "0.5"
 tokio        = "1.37"
 uuid         = "1.8"

packages/backend/server/src/core/doc/storage/doc.ts

@@ -1,8 +1,10 @@
 import {
   applyUpdate,
+  diffUpdate,
   Doc,
   encodeStateAsUpdate,
   encodeStateVector,
+  encodeStateVectorFromUpdate,
   mergeUpdates,
   UndoManager,
 } from 'yjs';
@@ -19,6 +21,12 @@ export interface DocRecord {
   editor?: string;
 }
 
+export interface DocDiff {
+  missing: Uint8Array;
+  state: Uint8Array;
+  timestamp: number;
+}
+
 export interface DocUpdate {
   bin: Uint8Array;
   timestamp: number;
@@ -96,6 +104,27 @@ export abstract class DocStorageAdapter extends Connection {
     return snapshot;
   }
 
+  async getDocDiff(
+    spaceId: string,
+    docId: string,
+    stateVector?: Uint8Array
+  ): Promise<DocDiff | null> {
+    const doc = await this.getDoc(spaceId, docId);
+
+    if (!doc) {
+      return null;
+    }
+
+    const missing = stateVector ? diffUpdate(doc.bin, stateVector) : doc.bin;
+    const state = encodeStateVectorFromUpdate(doc.bin);
+
+    return {
+      missing,
+      state,
+      timestamp: doc.timestamp,
+    };
+  }
+
   abstract pushDocUpdates(
     spaceId: string,
     docId: string,

packages/backend/server/src/core/doc/storage/index.ts

@@ -1,4 +1,6 @@
-// TODO(@forehalo): share with frontend
+// This is a totally copy of definitions in [@affine/doc-storage]
+// because currently importing cross workspace package from [@affine/server] is not yet supported
+// should be kept updated with the original definitions in [@affine/doc-storage]
 import type { BlobStorageAdapter } from './blob';
 import { Connection } from './connection';
 import type { DocStorageAdapter } from './doc';

packages/backend/server/src/core/sync/gateway.ts

@@ -8,7 +8,6 @@ import {
   WebSocketGateway,
 } from '@nestjs/websockets';
 import { Socket } from 'socket.io';
-import { diffUpdate, encodeStateVectorFromUpdate } from 'yjs';
 
 import {
   AlreadyInSpace,
@@ -233,31 +232,25 @@ export class SpaceSyncGateway
     @MessageBody()
     { spaceType, spaceId, docId, stateVector }: LoadDocMessage
   ): Promise<
-    EventResponse<{ missing: string; state?: string; timestamp: number }>
+    EventResponse<{ missing: string; state: string; timestamp: number }>
   > {
     const adapter = this.selectAdapter(client, spaceType);
     adapter.assertIn(spaceId);
 
-    const doc = await adapter.get(spaceId, docId);
+    const doc = await adapter.diff(
+      spaceId,
+      docId,
+      stateVector ? Buffer.from(stateVector, 'base64') : undefined
+    );
 
     if (!doc) {
       throw new DocNotFound({ spaceId, docId });
     }
 
-    const missing = Buffer.from(
-      stateVector
-        ? diffUpdate(doc.bin, Buffer.from(stateVector, 'base64'))
-        : doc.bin
-    ).toString('base64');
-
-    const state = Buffer.from(encodeStateVectorFromUpdate(doc.bin)).toString(
-      'base64'
-    );
-
     return {
       data: {
-        missing,
-        state,
+        missing: Buffer.from(doc.missing).toString('base64'),
+        state: Buffer.from(doc.state).toString('base64'),
         timestamp: doc.timestamp,
       },
     };
@@ -600,9 +593,9 @@ abstract class SyncSocketAdapter {
     return this.storage.pushDocUpdates(spaceId, docId, updates, editorId);
   }
 
-  get(spaceId: string, docId: string) {
+  diff(spaceId: string, docId: string, stateVector?: Uint8Array) {
     this.assertIn(spaceId);
-    return this.storage.getDoc(spaceId, docId);
+    return this.storage.getDocDiff(spaceId, docId, stateVector);
   }
 
   getTimestamps(spaceId: string, timestamp?: number) {
@@ -630,9 +623,9 @@ class WorkspaceSyncAdapter extends SyncSocketAdapter {
     return super.push(spaceId, id.guid, updates, editorId);
   }
 
-  override get(spaceId: string, docId: string) {
+  override diff(spaceId: string, docId: string, stateVector?: Uint8Array) {
     const id = new DocID(docId, spaceId);
-    return this.storage.getDoc(spaceId, id.guid);
+    return this.storage.getDocDiff(spaceId, id.guid, stateVector);
   }
 
   async assertAccessible(

packages/common/doc-storage/package.json

@@ -0,0 +1,22 @@
+{
+  "name": "@affine/doc-storage",
+  "type": "module",
+  "version": "0.15.0",
+  "private": true,
+  "sideEffects": false,
+  "exports": {
+    ".": "./index.ts",
+    "./impls/*": "./impls/*",
+    "./storage": "./storage/index.ts"
+  },
+  "dependencies": {
+    "@affine/native": "workspace:*",
+    "idb": "^8.0.0",
+    "lodash-es": "^4.17.21",
+    "socket.io-client": "^4.7.5",
+    "yjs": "patch:yjs@npm%3A13.6.18#~/.yarn/patches/yjs-npm-13.6.18-ad0d5f7c43.patch"
+  },
+  "devDependencies": {
+    "@types/lodash-es": "^4.17.12"
+  }
+}