Partially get key backups working on JS

- Modify the key backup test so client A is the backup creator and
  client B is the backup restorer. As such they must be on the same HS.
- Only skip the matrix if the backup restorer is JS, as that doesn't work.
  JS can be the backup creator though.
- Add globals Buffer and decodeRecoveryKey to `window` because ugh.
- tcpdump: include mitmproxy urls.
- Add some `cryptoCallbacks` when making JS clients.

README.md

@@ -103,7 +103,7 @@ Prerequisites:
 This repo has bindings to the `matrix_sdk` crate in rust SDK, in order to mimic Element X.
 
 In order to generate these bindings, follow these instructions:
-- Check out https://github.com/matrix-org/matrix-rust-sdk/tree/kegan/poljar-recovery-complement-fork (TODO: go back to main when
+- Check out https://github.com/matrix-org/matrix-rust-sdk/tree/kegan/complement-crypto (TODO: go back to main when
 main uses a versioned uniffi release e.g 0.25.2)
 - Get the bindings generator:
 ```
@@ -118,6 +118,7 @@ cargo install uniffi-bindgen-go --path ./uniffi-bindgen-go/bindgen
 - Generate the Go bindings to `./rust`: `uniffi-bindgen-go -l ../matrix-rust-sdk/target/debug/libmatrix_sdk_ffi.a -o ./rust ../matrix-rust-sdk/bindings/matrix-sdk-ffi/src/api.udl`
 - Patch up the generated code as it's not quite right:
     * Add `// #cgo LDFLAGS: -lmatrix_sdk_ffi` immediately after `// #include <matrix_sdk_ffi.h>` at the top of `matrix_sdk_ffi.go`.
+    * https://github.com/NordSecurity/uniffi-bindgen-go/issues/36
 - Sanity check compile `LIBRARY_PATH="$LIBRARY_PATH:/path/to/matrix-rust-sdk/target/debug" go test -c ./tests`
 
 

internal/api/js.go

@@ -65,6 +65,7 @@ func NewJSClient(t *testing.T, opts ClientCreationOpts) (Client, error) {
 	ctx, cancel := chromedp.NewContext(context.Background(), chromedp.WithBrowserOption(
 		chromedp.WithBrowserLogf(colorifyError), chromedp.WithBrowserErrorf(colorifyError), //chromedp.WithBrowserDebugf(log.Printf),
 	))
+
 	jsc := &JSClient{
 		listeners: make(map[int32]func(roomID string, ev Event)),
 		userID:    opts.UserID,
@@ -79,7 +80,7 @@ func NewJSClient(t *testing.T, opts ClientCreationOpts) (Client, error) {
 					s = string(arg.Value)
 				}
 				// TODO: debug mode only?
-				writeToLog("[%s] console.log %s\n", opts.UserID, s)
+				writeToLog("[%s,%s] console.log %s\n", jsc.baseJSURL, opts.UserID, s)
 
 				if strings.HasPrefix(s, CONSOLE_LOG_CONTROL_STRING) {
 					val := strings.TrimPrefix(s, CONSOLE_LOG_CONTROL_STRING)
@@ -141,7 +142,35 @@ func NewJSClient(t *testing.T, opts ClientCreationOpts) (Client, error) {
 	if err != nil {
 		return nil, fmt.Errorf("failed to serialise login info: %s", err)
 	}
-	val := fmt.Sprintf("window.__client = matrix.createClient(%s);", string(createClientOptsJSON))
+	// inject crypto callback functions, which need to be done without json serialisation :/
+	// start with '{' then [1:] the JSON to inject well-formed JS objects
+	args := `{
+			cryptoCallbacks: {
+				cacheSecretStorageKey: (keyId, keyInfo, key) => {
+					console.log("cacheSecretStorageKey: keyId="+keyId+" keyInfo="+JSON.stringify(keyInfo)+" key.length:"+key.length);
+					window._secretStorageKeys[keyId] = {
+						keyInfo: keyInfo,
+						key: key,
+					};
+				},
+				getSecretStorageKey: (keys, name) => { //
+					console.log("getSecretStorageKey: name=" + name + " keys=" + JSON.stringify(keys));
+					const result = [];
+					for (const keyId of Object.keys(keys.keys)) {
+						const ssKey = window._secretStorageKeys[keyId];
+						if (ssKey) {
+							result.push(keyId);
+							result.push(ssKey.key);
+							console.log("getSecretStorageKey: found key ID: " + keyId);
+						} else {
+							console.log("getSecretStorageKey: unknown key ID: " + keyId);
+						}
+					}
+					return Promise.resolve(result);
+				},
+			},
+			` + string(createClientOptsJSON[1:])
+	val := fmt.Sprintf("window._secretStorageKeys = {}; window.__client = matrix.createClient(%s);", args)
 	fmt.Println(val)
 	// TODO: move to chrome package
 	var r *runtime.RemoteObject
@@ -306,12 +335,42 @@ func (c *JSClient) MustBackpaginate(t *testing.T, roomID string, count int) {
 }
 
 func (c *JSClient) MustBackupKeys(t *testing.T) (recoveryKey string) {
-	// TODO
-	return
+	key, err := chrome.AwaitExecuteInto[string](t, c.ctx, `(async () => {
+		// we need to ensure that we have a recovery key first, though we don't actually care about it..?
+		const recoveryKey = await window.__client.getCrypto().createRecoveryKeyFromPassphrase();
+		// now use said key to make backups
+		await window.__client.getCrypto().bootstrapSecretStorage({
+			createSecretStorageKey: async() => { return recoveryKey; },
+			setupNewKeyBackup: true,
+			setupNewSecretStorage: true,
+		});
+		// now we can enable key backups
+		await window.__client.getCrypto().checkKeyBackupAndEnable();
+		return recoveryKey.encodedPrivateKey;
+	})()`)
+	if err != nil {
+		fatalf(t, "MustBackupKeys: %s", err)
+	}
+	time.Sleep(time.Second)
+	return *key
 }
 
 func (c *JSClient) MustLoadBackup(t *testing.T, recoveryKey string) {
-	// TODO
+	chrome.MustAwaitExecute(t, c.ctx, fmt.Sprintf(`(async () => {
+		// add the recovery key to secret storage
+		const recoveryKeyInfo = await window.__client.secretStorage.addKey("m.secret_storage.v1.aes-hmac-sha2", {
+			key: window.decodeRecoveryKey("%s"),
+		});
+		console.log("setting default key ID to " + recoveryKeyInfo.keyId);
+		// FIXME: this needs the client to be syncing already as this promise won't resolve until it comes down /sync, wedging forever
+		await window.__client.secretStorage.setDefaultKeyId(recoveryKeyInfo.keyId);
+		console.log("done!");
+		const keyBackupCheck = await window.__client.getCrypto().checkKeyBackupAndEnable();
+		console.log("key backup: ", JSON.stringify(keyBackupCheck));
+		// FIXME: this just doesn't seem to work, causing 'Error: getSecretStorageKey callback returned invalid data' because the key ID
+		// cannot be found...
+		await window.__client.restoreKeyBackupWithSecretStorage(keyBackupCheck ? keyBackupCheck.backupInfo : null, undefined, undefined);
+	})()`, recoveryKey))
 }
 
 func (c *JSClient) WaitUntilEventInRoom(t *testing.T, roomID string, checker func(e Event) bool) Waiter {

internal/deploy/deploy.go

@@ -235,10 +235,15 @@ func RunNewDeployment(t *testing.T, shouldTCPDump bool) *SlidingSyncDeployment {
 	var cmd *exec.Cmd
 	if shouldTCPDump {
 		t.Log("Running tcpdump...")
-		su, _ := url.Parse(ssURL)
-		cu1, _ := url.Parse(csapi1.BaseURL)
-		cu2, _ := url.Parse(csapi2.BaseURL)
-		filter := fmt.Sprintf("tcp port %s or port %s or port %s", su.Port(), cu1.Port(), cu2.Port())
+		urlsToTCPDump := []string{
+			ssURL, csapi1.BaseURL, csapi2.BaseURL, rpHS1URL, rpHS2URL, controllerURL,
+		}
+		tcpdumpFilter := []string{}
+		for _, u := range urlsToTCPDump {
+			parsedURL, _ := url.Parse(u)
+			tcpdumpFilter = append(tcpdumpFilter, fmt.Sprintf("port %s", parsedURL.Port()))
+		}
+		filter := fmt.Sprintf("tcp " + strings.Join(tcpdumpFilter, " or "))
 		cmd = exec.Command("tcpdump", "-i", "any", "-s", "0", filter, "-w", "test.pcap")
 		t.Log(cmd.String())
 		if err := cmd.Start(); err != nil {

js-sdk/index.html

@@ -3,6 +3,10 @@
 
 <head>
   <script type="module">
+    import { Buffer } from "buffer";
+    window.Buffer = Buffer;
+    import { decodeRecoveryKey } from "matrix-js-sdk/src/crypto/recoverykey";
+    window.decodeRecoveryKey = decodeRecoveryKey;
     import * as sdk from "matrix-js-sdk";
     window.matrix = sdk;
   </script>

js-sdk/package.json

@@ -9,7 +9,8 @@
   "author": "",
   "license": "Apache-2.0",
   "dependencies": {
-    "matrix-js-sdk": "^30.3.0",
+    "buffer": "^6.0.3",
+    "matrix-js-sdk": "https://github.com/matrix-org/matrix-js-sdk#febef3fc7c67ec9e3cb5103f52914013e91cf59c",
     "vite": "^4.5.0"
   }
 }

js-sdk/vite.config.js

@@ -2,6 +2,10 @@
 import path from 'path';
 
 export default {
+  // Node.js global to browser globalThis
+  define: {
+    global: 'globalThis',
+},
   build: {
     // disabled because https://github.com/matrix-org/matrix-rust-sdk-crypto-wasm/issues/51
     minify: false,

js-sdk/yarn.lock

@@ -119,7 +119,7 @@
   resolved "https://registry.yarnpkg.com/@esbuild/win32-x64/-/win32-x64-0.18.20.tgz#786c5f41f043b07afb1af37683d7c33668858f6d"
   integrity sha512-kTdfRcSiDfQca/y9QIkng02avJ+NCaQvrMejlsB3RRv5sE9rRoeBPISaZpKxHELzRxZyLvNts1P27W3wV+8geQ==
 
-"@matrix-org/matrix-sdk-crypto-wasm@^3.4.0":
+"@matrix-org/matrix-sdk-crypto-wasm@^3.5.0":
   version "3.5.0"
   resolved "https://registry.yarnpkg.com/@matrix-org/matrix-sdk-crypto-wasm/-/matrix-sdk-crypto-wasm-3.5.0.tgz#997d63ae12304142513fe93c5e0872ff10ca30b4"
   integrity sha512-7as0jJTje+rFu9AF8LEO0tmhtHcou2YQnZOtpiP+lS5rDfIPv5CL8/eb45fzDnbQybt9Jm5zdjBdiLBEaUg2dQ==
@@ -144,13 +144,26 @@ base-x@^4.0.0:
   resolved "https://registry.yarnpkg.com/base-x/-/base-x-4.0.0.tgz#d0e3b7753450c73f8ad2389b5c018a4af7b2224a"
   integrity sha512-FuwxlW4H5kh37X/oW59pwTzzTKRzfrrQwhmyspRM7swOEZcHtDZSCt45U6oKgtuFE+WYPblePMVIPR4RZrh/hw==
 
+base64-js@^1.3.1:
+  version "1.5.1"
+  resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.5.1.tgz#1b1b440160a5bf7ad40b650f095963481903930a"
+  integrity sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==
+
 bs58@^5.0.0:
   version "5.0.0"
   resolved "https://registry.yarnpkg.com/bs58/-/bs58-5.0.0.tgz#865575b4d13c09ea2a84622df6c8cbeb54ffc279"
   integrity sha512-r+ihvQJvahgYT50JD05dyJNKlmmSlMoOGwn1lCcEzanPglg7TxYjioQUYehQ9mAR/+hOSd2jRc/Z2y5UxBymvQ==
   dependencies:
     base-x "^4.0.0"
 
+buffer@^6.0.3:
+  version "6.0.3"
+  resolved "https://registry.yarnpkg.com/buffer/-/buffer-6.0.3.tgz#2ace578459cc8fbe2a70aaa8f52ee63b6a74c6c6"
+  integrity sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==
+  dependencies:
+    base64-js "^1.3.1"
+    ieee754 "^1.2.1"
+
 content-type@^1.0.4:
   version "1.0.5"
   resolved "https://registry.yarnpkg.com/content-type/-/content-type-1.0.5.tgz#8b773162656d1d1086784c8f23a54ce6d73d7918"
@@ -199,6 +212,11 @@ fsevents@~2.3.2:
   resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.3.3.tgz#cac6407785d03675a2a5e1a5305c697b347d90d6"
   integrity sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==
 
+ieee754@^1.2.1:
+  version "1.2.1"
+  resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.2.1.tgz#8eb7a10a63fff25d15a57b001586d177d1b0d352"
+  integrity sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==
+
 jwt-decode@^3.1.2:
   version "3.1.2"
   resolved "https://registry.yarnpkg.com/jwt-decode/-/jwt-decode-3.1.2.tgz#3fb319f3675a2df0c2895c8f5e9fa4b67b04ed59"
@@ -214,13 +232,12 @@ [email protected]:
   resolved "https://registry.yarnpkg.com/matrix-events-sdk/-/matrix-events-sdk-0.0.1.tgz#c8c38911e2cb29023b0bbac8d6f32e0de2c957dd"
   integrity sha512-1QEOsXO+bhyCroIe2/A5OwaxHvBm7EsSQ46DEDn8RBIfQwN5HWBpFvyWWR4QY0KHPPnnJdI99wgRiAl7Ad5qaA==
 
-matrix-js-sdk@^30.3.0:
+"matrix-js-sdk@https://github.com/matrix-org/matrix-js-sdk#febef3fc7c67ec9e3cb5103f52914013e91cf59c":
   version "30.3.0"
-  resolved "https://registry.yarnpkg.com/matrix-js-sdk/-/matrix-js-sdk-30.3.0.tgz#f7046da2d6d6403c84aac46b16772248e068fb9e"
-  integrity sha512-yqAn1IhvrSxvqRP4UMToaWhtA/iC6FYTt4qj5K8H3BmAQDOqObw9qPLm43HmdbsBGk6VUwz9szgNblhVyq0sKg==
+  resolved "https://github.com/matrix-org/matrix-js-sdk#febef3fc7c67ec9e3cb5103f52914013e91cf59c"
   dependencies:
     "@babel/runtime" "^7.12.5"
-    "@matrix-org/matrix-sdk-crypto-wasm" "^3.4.0"
+    "@matrix-org/matrix-sdk-crypto-wasm" "^3.5.0"
     another-json "^0.2.0"
     bs58 "^5.0.0"
     content-type "^1.0.4"

tests/key_backup_test.go

@@ -5,58 +5,78 @@ import (
 	"time"
 
 	"github.com/matrix-org/complement-crypto/internal/api"
+	"github.com/matrix-org/complement/helpers"
 	"github.com/matrix-org/complement/must"
 )
 
+// TODO: client types should be bob 1 and bob 2, NOT alice who is just used to send an encrypted msg.
+// This allows us to test that backups made on FFI can be read on JS and vice versa.
 func TestCanBackupKeys(t *testing.T) {
 	ClientTypeMatrix(t, func(t *testing.T, clientTypeA, clientTypeB api.ClientType) {
 		if clientTypeB.Lang == api.ClientTypeJS {
-			t.Skipf("key backups unsupported (js)")
+			t.Skipf("key backup restoring is unsupported (js)")
 			return
 		}
-		tc := CreateTestContext(t, clientTypeA, clientTypeB)
-		// shared history visibility
-		roomID := tc.CreateNewEncryptedRoom(t, tc.Alice, "public_chat", nil)
-		tc.Bob.MustJoinRoom(t, roomID, []string{clientTypeA.HS})
+		if clientTypeA.HS != clientTypeB.HS {
+			t.Skipf("client A and B must be on the same HS as this is testing key backups so A=backup creator B=backup restorer")
+			return
+		}
+		deployment := Deploy(t)
+		csapiAlice := deployment.Register(t, clientTypeA.HS, helpers.RegistrationOpts{
+			LocalpartSuffix: "alice",
+			Password:        "complement-crypto-password",
+		})
+		roomID := csapiAlice.MustCreateRoom(t, map[string]interface{}{
+			"name":   t.Name(),
+			"preset": "public_chat", // shared history visibility
+			"invite": []string{},
+			"initial_state": []map[string]interface{}{
+				{
+					"type":      "m.room.encryption",
+					"state_key": "",
+					"content": map[string]interface{}{
+						"algorithm": "m.megolm.v1.aes-sha2",
+					},
+				},
+			},
+		})
 
 		// SDK testing below
 		// -----------------
 
-		// login both clients first, so OTKs etc are uploaded.
-		alice := tc.MustLoginClient(t, tc.Alice, clientTypeA)
-		defer alice.Close(t)
-		bob := tc.MustLoginClient(t, tc.Bob, clientTypeB)
-		defer bob.Close(t)
-
-		// Alice and Bob start syncing
-		aliceStopSyncing := alice.StartSyncing(t)
-		defer aliceStopSyncing()
-		bobStopSyncing := bob.StartSyncing(t)
-		defer bobStopSyncing()
+		backupCreator := LoginClientFromComplementClient(t, deployment, csapiAlice, clientTypeA)
+		defer backupCreator.Close(t)
+		stopSyncing := backupCreator.StartSyncing(t)
+		defer stopSyncing()
 
-		// Alice sends a message which Bob should be able to decrypt
 		body := "An encrypted message"
-		waiter := bob.WaitUntilEventInRoom(t, roomID, api.CheckEventHasBody(body))
-		evID := alice.SendMessage(t, roomID, body)
-		t.Logf("bob (%s) waiting for event %s", bob.Type(), evID)
+		waiter := backupCreator.WaitUntilEventInRoom(t, roomID, api.CheckEventHasBody(body))
+		evID := backupCreator.SendMessage(t, roomID, body)
+		t.Logf("backupCreator (%s) waiting for event %s", backupCreator.Type(), evID)
 		waiter.Wait(t, 5*time.Second)
 
-		// Now Bob backs up his keys. Some clients may automatically do this, but let's be explicit about it.
-		recoveryKey := bob.MustBackupKeys(t)
+		// Now backupCreator backs up his keys. Some clients may automatically do this, but let's be explicit about it.
+		recoveryKey := backupCreator.MustBackupKeys(t)
+		t.Logf("recovery key -> %s", recoveryKey)
 
-		// Now Bob logs in on a new device
-		_, bob2 := tc.MustLoginDevice(t, tc.Bob, clientTypeB, "NEW_DEVICE")
+		// Now login on a new device
+		csapiAlice2 := deployment.Login(t, clientTypeB.HS, csapiAlice, helpers.LoginOpts{
+			DeviceID: "BACKUP_RESTORER",
+			Password: "complement-crypto-password",
+		})
+		backupRestorer := LoginClientFromComplementClient(t, deployment, csapiAlice2, clientTypeB)
+		defer backupRestorer.Close(t)
 
-		// Bob loads the key backup using the recovery key
-		bob2.MustLoadBackup(t, recoveryKey)
+		// load the key backup using the recovery key
+		backupRestorer.MustLoadBackup(t, recoveryKey)
 
-		// Bob's new device can decrypt the encrypted message
-		bob2StopSyncing := bob2.StartSyncing(t)
-		defer bob2StopSyncing()
+		// new device can decrypt the encrypted message
+		backupRestorerStopSyncing := backupRestorer.StartSyncing(t)
+		defer backupRestorerStopSyncing()
 		time.Sleep(time.Second)
-		bob2.MustBackpaginate(t, roomID, 5) // get the old message
+		backupRestorer.MustBackpaginate(t, roomID, 5) // get the old message
 
-		ev := bob2.MustGetEvent(t, roomID, evID)
+		ev := backupRestorer.MustGetEvent(t, roomID, evID)
 		must.Equal(t, ev.FailedToDecrypt, false, "bob's new device failed to decrypt the event: bad backup?")
 		must.Equal(t, ev.Text, body, "bob's new device failed to see the clear text message")
 	})

tests/main_test.go

@@ -157,8 +157,12 @@ func (c *TestContext) MustLoginDevice(t *testing.T, existing *client.CSAPI, clie
 }
 
 func (c *TestContext) MustLoginClient(t *testing.T, cli *client.CSAPI, clientType api.ClientType) api.Client {
+	return LoginClientFromComplementClient(t, c.Deployment, cli, clientType)
+}
+
+func LoginClientFromComplementClient(t *testing.T, dep *deploy.SlidingSyncDeployment, cli *client.CSAPI, clientType api.ClientType) api.Client {
 	t.Helper()
 	cfg := api.FromComplementClient(cli, "complement-crypto-password")
-	cfg.BaseURL = c.Deployment.ReverseProxyURLForHS(clientType.HS)
-	return MustLoginClient(t, clientType, cfg, c.Deployment.SlidingSyncURL(t))
+	cfg.BaseURL = dep.ReverseProxyURLForHS(clientType.HS)
+	return MustLoginClient(t, clientType, cfg, dep.SlidingSyncURL(t))
 }