diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 395e7ffb..cd656544 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -18,9 +18,6 @@ jobs: - uses: actions/setup-go@v4 with: go-version-file: go.mod - - name: "Install Complement Dependencies" - run: | - sudo apt-get update && sudo apt-get install -y libolm3 libolm-dev - name: "Run internal Complement tests" run: | go test ./internal/... @@ -69,7 +66,6 @@ jobs: # servers which listen on random high numbered ports. - name: "Install Complement Dependencies" run: | - sudo apt-get update && sudo apt-get install -y libolm3 libolm-dev go install -v github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt@latest mkdir .gotestfmt/github -p cp .ci/complement_package.gotpl .gotestfmt/github/package.gotpl diff --git a/b/blueprints.go b/b/blueprints.go index 9b2ce150..1c8fc4c7 100644 --- a/b/blueprints.go +++ b/b/blueprints.go @@ -32,7 +32,6 @@ var KnownBlueprints = map[string]*Blueprint{ BlueprintOneToOneRoom.Name: &BlueprintOneToOneRoom, BlueprintPerfManyMessages.Name: &BlueprintPerfManyMessages, BlueprintPerfManyRooms.Name: &BlueprintPerfManyRooms, - BlueprintPerfE2EERoom.Name: &BlueprintPerfE2EERoom, } // Blueprint represents an entire deployment to make. @@ -64,10 +63,6 @@ type User struct { AvatarURL string AccountData []AccountData DeviceID *string - // Enable end-to end encryption for this user and upload the given - // amount of one-time keys. This requires the DeviceId to be set as - // well. - OneTimeKeys uint } type AccountData struct { diff --git a/b/perf_e2ee_room.go b/b/perf_e2ee_room.go deleted file mode 100644 index ae171b22..00000000 --- a/b/perf_e2ee_room.go +++ /dev/null @@ -1,104 +0,0 @@ -package b - -import "fmt" - -var userCount = 500 - -// BlueprintOneToOneRoom contains a homeserver with 500 E2EE capable users, -// who are joined to the same room. -var BlueprintPerfE2EERoom = MustValidate(Blueprint{ - Name: "perf_e2ee_room", - Homeservers: []Homeserver{ - { - Name: "hs1", - Users: append([]User{ - { - Localpart: "@alice", - DisplayName: "Alice", - OneTimeKeys: 50, - DeviceID: Ptr("ALDJLSKJD"), - }, - { - Localpart: "@bob", - DisplayName: "Bob", - DeviceID: Ptr("BOBDASLDKJ"), - }, - }, manyUsers(userCount)...), - Rooms: []Room{ - { - CreateRoom: map[string]interface{}{ - "preset": "public_chat", - }, - Creator: "@alice", - Events: append([]Event{ - { - Type: "m.room.member", - StateKey: Ptr("@bob:hs1"), - Content: map[string]interface{}{ - "membership": "join", - }, - Sender: "@bob", - }, - { - Type: "m.room.message", - Content: map[string]interface{}{ - "body": "Hello world", - "msgtype": "m.text", - }, - Sender: "@bob", - }, - { - Type: "m.room.encryption", - StateKey: Ptr(""), - Content: map[string]interface{}{ - "algorithm": "m.megolm.v1.aes-sha2", - }, - Sender: "@alice", - }, - }, memberships(userCount)...), - }, - }, - }, - }, -}) - -func memberships(count int) []Event { - events := make([]Event, count) - - for i := 0; i < count; i++ { - localPart := fmt.Sprintf("@alice_%d", i) - stateKey := fmt.Sprintf("%s:hs1", localPart) - - event := Event{ - Type: "m.room.member", - StateKey: Ptr(stateKey), - Content: map[string]interface{}{ - "membership": "join", - }, - Sender: localPart, - } - - events[i] = event - } - - return events -} - -func manyUsers(count int) []User { - users := make([]User, count) - - for i := 0; i < count; i++ { - localPart := fmt.Sprintf("@alice_%d", i) - displayName := fmt.Sprintf("Alice %d", i) - deviceID := fmt.Sprintf("ALICEDEVICE%d", i) - - users[i] = User{ - Localpart: localPart, - DisplayName: displayName, - OneTimeKeys: 50, - DeviceID: Ptr(deviceID), - } - } - - return users -} diff --git a/client/client.go b/client/client.go index d98bb6e1..916cf324 100644 --- a/client/client.go +++ b/client/client.go @@ -3,9 +3,12 @@ package client import ( "bytes" "context" // nolint:gosec + "crypto/ed25519" + "encoding/base64" "encoding/json" "fmt" "io" + "math/rand" "net/http" "net/http/httputil" "net/url" @@ -16,7 +19,7 @@ import ( "github.com/matrix-org/gomatrixserverlib" "github.com/tidwall/gjson" - "maunium.net/go/mautrix/crypto/olm" + "golang.org/x/crypto/curve25519" "github.com/matrix-org/complement/b" "github.com/matrix-org/complement/ct" @@ -28,6 +31,12 @@ const ( CtxKeyWithRetryUntil ctxKey = "complement_retry_until" // contains *retryUntilParams ) +var ( + // use a deterministic seed but globally so we don't generate the same numbers for each client. + // This could be non-deterministic if used concurrently. + prng = rand.New(rand.NewSource(42)) +) + type retryUntilParams struct { timeout time.Duration untilFn func(*http.Response) bool @@ -403,10 +412,21 @@ func (c *CSAPI) MustUploadKeys(t ct.TestLike, deviceKeys map[string]interface{}, return s.OTKCounts } +// Generate realistic looking device keys and OTKs. They are not guaranteed to be 100% valid, but should +// pass most server-side checks. Critically, these keys are generated using a Pseudo-Random Number Generator (PRNG) +// for determinism and hence ARE NOT SECURE. DO NOT USE THIS OUTSIDE OF TESTS. func (c *CSAPI) MustGenerateOneTimeKeys(t ct.TestLike, otkCount uint) (deviceKeys map[string]interface{}, oneTimeKeys map[string]interface{}) { t.Helper() - account := olm.NewAccount() - ed25519Key, curveKey := account.IdentityKeys() + ed25519PubKey, ed25519PrivKey, err := ed25519.GenerateKey(prng) + if err != nil { + ct.Fatalf(t, "failed to generate ed25519 key: %s", err) + } + + curveKey := make([]byte, 32) + _, err = prng.Read(curveKey) + if err != nil { + ct.Fatalf(t, "failed to read from prng: %s", err) + } ed25519KeyID := fmt.Sprintf("ed25519:%s", c.DeviceID) curveKeyID := fmt.Sprintf("curve25519:%s", c.DeviceID) @@ -416,38 +436,59 @@ func (c *CSAPI) MustGenerateOneTimeKeys(t ct.TestLike, otkCount uint) (deviceKey "device_id": c.DeviceID, "algorithms": []interface{}{"m.olm.v1.curve25519-aes-sha2", "m.megolm.v1.aes-sha2"}, "keys": map[string]interface{}{ - ed25519KeyID: ed25519Key.String(), - curveKeyID: curveKey.String(), + ed25519KeyID: base64.RawStdEncoding.EncodeToString(ed25519PubKey), + curveKeyID: base64.RawStdEncoding.EncodeToString(curveKey), }, } - signature, _ := account.SignJSON(deviceKeys) + signJSON := func(input any) []byte { + inputJSON, err := json.Marshal(input) + if err != nil { + ct.Fatalf(t, "failed to marshal struct: %s", err) + } + inputJSON, err = gomatrixserverlib.CanonicalJSON(inputJSON) + if err != nil { + ct.Fatalf(t, "failed to canonical json: %s", err) + } + signature := ed25519.Sign(ed25519PrivKey, inputJSON) + if err != nil { + ct.Fatalf(t, "failed to sign json: %s", err) + } + return signature + } deviceKeys["signatures"] = map[string]interface{}{ c.UserID: map[string]interface{}{ - ed25519KeyID: signature, + ed25519KeyID: base64.RawStdEncoding.EncodeToString(signJSON(deviceKeys)), }, } - - account.GenOneTimeKeys(otkCount) oneTimeKeys = map[string]interface{}{} - for kid, key := range account.OneTimeKeys() { + for i := uint(0); i < otkCount; i++ { + privateKeyBytes := make([]byte, 32) + _, err = prng.Read(privateKeyBytes) + if err != nil { + ct.Fatalf(t, "failed to read from prng", err) + } + key, err := curve25519.X25519(privateKeyBytes, curve25519.Basepoint) + if err != nil { + ct.Fatalf(t, "failed to generate curve pubkey: %s", err) + } + kid := fmt.Sprintf("%d", i) keyID := fmt.Sprintf("signed_curve25519:%s", kid) keyMap := map[string]interface{}{ - "key": key.String(), + "key": base64.RawStdEncoding.EncodeToString(key), } - signature, _ = account.SignJSON(keyMap) - keyMap["signatures"] = map[string]interface{}{ c.UserID: map[string]interface{}{ - ed25519KeyID: signature, + ed25519KeyID: base64.RawStdEncoding.EncodeToString(signJSON(keyMap)), }, } oneTimeKeys[keyID] = keyMap } + return deviceKeys, oneTimeKeys } diff --git a/cmd/account-snapshot/internal/blueprint.go b/cmd/account-snapshot/internal/blueprint.go index 92c0a636..989e2b50 100644 --- a/cmd/account-snapshot/internal/blueprint.go +++ b/cmd/account-snapshot/internal/blueprint.go @@ -40,10 +40,6 @@ func ConvertToBlueprint(s *Snapshot, serverName string) (*b.Blueprint, error) { DisplayName: local, DeviceID: &devices[i], } - // set up OTKs if this user+device sends E2E messages - if devices[i] != NoEncryptedDevice { - user.OneTimeKeys = 100 - } // set DM list if this is the syncing user if userID == s.UserID { user.AccountData = append(user.AccountData, b.AccountData{ diff --git a/go.mod b/go.mod index 9ed49272..7723a1fe 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,6 @@ require ( github.com/tidwall/sjson v1.2.5 golang.org/x/exp v0.0.0-20230905200255-921286631fa9 gonum.org/v1/plot v0.11.0 - maunium.net/go/mautrix v0.11.0 ) require ( diff --git a/internal/instruction/runner.go b/internal/instruction/runner.go index 97394b96..16db45f5 100644 --- a/internal/instruction/runner.go +++ b/internal/instruction/runner.go @@ -16,7 +16,6 @@ import ( "time" "github.com/tidwall/gjson" - "maunium.net/go/mautrix/crypto/olm" "github.com/matrix-org/complement/b" ) @@ -418,9 +417,6 @@ func calculateUserInstructionSets(r *Runner, hs b.Homeserver) [][]instruction { } createdUsers[user.Localpart] = true - if user.OneTimeKeys > 0 { - instrs = append(instrs, instructionOneTimeKeyUpload(hs, user)) - } sets[i] = instrs } return sets @@ -636,65 +632,6 @@ func instructionLogin(hs b.Homeserver, user b.User) instruction { } } -func instructionOneTimeKeyUpload(hs b.Homeserver, user b.User) instruction { - account := olm.NewAccount() - ed25519Key, curveKey := account.IdentityKeys() - - userID := fmt.Sprintf("@%s:%s", user.Localpart, hs.Name) - deviceID := *user.DeviceID - - ed25519KeyID := fmt.Sprintf("ed25519:%s", deviceID) - curveKeyID := fmt.Sprintf("curve25519:%s", deviceID) - - deviceKeys := map[string]interface{}{ - "user_id": userID, - "device_id": deviceID, - "algorithms": []string{"m.olm.v1.curve25519-aes-sha2", "m.megolm.v1.aes-sha2"}, - "keys": map[string]string{ - ed25519KeyID: ed25519Key.String(), - curveKeyID: curveKey.String(), - }, - } - - signature, _ := account.SignJSON(deviceKeys) - - deviceKeys["signatures"] = map[string]map[string]string{ - userID: { - ed25519KeyID: signature, - }, - } - - account.GenOneTimeKeys(user.OneTimeKeys) - - oneTimeKeys := map[string]interface{}{} - - for kid, key := range account.OneTimeKeys() { - keyID := fmt.Sprintf("signed_curve25519:%s", kid) - keyMap := map[string]interface{}{ - "key": key.String(), - } - - signature, _ = account.SignJSON(keyMap) - - keyMap["signatures"] = map[string]interface{}{ - userID: map[string]string{ - ed25519KeyID: signature, - }, - } - - oneTimeKeys[keyID] = keyMap - } - return instruction{ - method: "POST", - path: "/_matrix/client/v3/keys/upload", - accessToken: fmt.Sprintf("user_@%s:%s", user.Localpart, hs.Name), - body: map[string]interface{}{ - "device_keys": deviceKeys, - "one_time_keys": oneTimeKeys, - }, - } -} - // indexFor hashes the input and returns a number % numEntries func indexFor(input string, numEntries int) int { hh := fnv.New32a() diff --git a/tests/csapi/device_lists_test.go b/tests/csapi/device_lists_test.go index e25dbbce..4c259f78 100644 --- a/tests/csapi/device_lists_test.go +++ b/tests/csapi/device_lists_test.go @@ -1,16 +1,18 @@ package csapi_tests import ( + "encoding/base64" "fmt" "testing" + "math/rand" + "github.com/matrix-org/complement" "github.com/matrix-org/complement/client" "github.com/matrix-org/complement/helpers" "github.com/matrix-org/complement/match" "github.com/matrix-org/complement/must" "github.com/matrix-org/complement/runtime" - "maunium.net/go/mautrix/crypto/olm" "github.com/tidwall/gjson" ) @@ -19,16 +21,22 @@ import ( // 1. `/sync`'s `device_lists.changed/left` contain the correct user IDs. // 2. `/keys/query` returns the correct information after device list updates. func TestDeviceListUpdates(t *testing.T) { + prng := rand.New(rand.NewSource(42)) // uploadNewKeys uploads a new set of keys for a given client. // Returns a check function that can be passed to mustQueryKeys. uploadNewKeys := func(t *testing.T, user *client.CSAPI) []match.JSON { t.Helper() - - account := olm.NewAccount() - ed25519Key, curve25519Key := account.IdentityKeys() - ed25519KeyID := fmt.Sprintf("ed25519:%s", user.DeviceID) curve25519KeyID := fmt.Sprintf("curve25519:%s", user.DeviceID) + // generate key-like looking values + ed25519KeyBytes := make([]byte, 32) + _, err := prng.Read(ed25519KeyBytes) + must.NotError(t, "failed to read from prng", err) + ed25519Key := base64.RawStdEncoding.EncodeToString(ed25519KeyBytes) + curve25519KeyBytes := make([]byte, 32) + _, err = prng.Read(curve25519KeyBytes) + must.NotError(t, "failed to read from prng", err) + curve25519Key := base64.RawStdEncoding.EncodeToString(curve25519KeyBytes) user.MustDo(t, "POST", []string{"_matrix", "client", "v3", "keys", "upload"}, client.WithJSONBody(t, map[string]interface{}{ @@ -37,8 +45,8 @@ func TestDeviceListUpdates(t *testing.T) { "device_id": user.DeviceID, "algorithms": []interface{}{"m.olm.v1.curve25519-aes-sha2", "m.megolm.v1.aes-sha2"}, "keys": map[string]interface{}{ - ed25519KeyID: ed25519Key.String(), - curve25519KeyID: curve25519Key.String(), + ed25519KeyID: ed25519Key, + curve25519KeyID: curve25519Key, }, }, }), @@ -49,8 +57,8 @@ func TestDeviceListUpdates(t *testing.T) { curve25519Path := fmt.Sprintf("device_keys.%s.%s.keys.%s", user.UserID, user.DeviceID, curve25519KeyID) return []match.JSON{ match.JSONKeyEqual(algorithmsPath, []interface{}{"m.olm.v1.curve25519-aes-sha2", "m.megolm.v1.aes-sha2"}), - match.JSONKeyEqual(ed25519Path, ed25519Key.String()), - match.JSONKeyEqual(curve25519Path, curve25519Key.String()), + match.JSONKeyEqual(ed25519Path, ed25519Key), + match.JSONKeyEqual(curve25519Path, curve25519Key), } }