From 2c58bab6a8727f56920b6c8b4d035bfcf9929744 Mon Sep 17 00:00:00 2001 From: Till Faelligen <2353100+S7evinK@users.noreply.github.com> Date: Wed, 15 Mar 2023 08:21:00 +0100 Subject: [PATCH 01/42] Fix UTs on x86 --- internal/transactionrequest_test.go | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/internal/transactionrequest_test.go b/internal/transactionrequest_test.go index 8597ae24b5..c152eb2856 100644 --- a/internal/transactionrequest_test.go +++ b/internal/transactionrequest_test.go @@ -22,6 +22,12 @@ import ( "testing" "time" + "github.com/matrix-org/gomatrixserverlib" + "github.com/nats-io/nats.go" + "github.com/stretchr/testify/assert" + "go.uber.org/atomic" + "gotest.tools/v3/poll" + "github.com/matrix-org/dendrite/federationapi/producers" rsAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/setup/config" @@ -30,11 +36,6 @@ import ( "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/dendrite/test" keyAPI "github.com/matrix-org/dendrite/userapi/api" - "github.com/matrix-org/gomatrixserverlib" - "github.com/nats-io/nats.go" - "github.com/stretchr/testify/assert" - "go.uber.org/atomic" - "gotest.tools/v3/poll" ) const ( @@ -427,7 +428,7 @@ func TestProcessTransactionRequestEDUReceipt(t *testing.T) { roomID: map[string]interface{}{ "m.read": map[string]interface{}{ "@john:kaer.morhen": map[string]interface{}{ - "data": map[string]interface{}{ + "data": map[string]int64{ "ts": 1533358089009, }, "event_ids": []string{ @@ -446,7 +447,7 @@ func TestProcessTransactionRequestEDUReceipt(t *testing.T) { roomID: map[string]interface{}{ "m.read": map[string]interface{}{ "johnkaer.morhen": map[string]interface{}{ - "data": map[string]interface{}{ + "data": map[string]int64{ "ts": 1533358089009, }, "event_ids": []string{ @@ -463,7 +464,7 @@ func TestProcessTransactionRequestEDUReceipt(t *testing.T) { roomID: map[string]interface{}{ "m.read": map[string]interface{}{ "@john:bad.domain": map[string]interface{}{ - "data": map[string]interface{}{ + "data": map[string]int64{ "ts": 1533358089009, }, "event_ids": []string{ From d88f71ab71a60348518f7fa6735ac9f0bfb472c3 Mon Sep 17 00:00:00 2001 From: Boris Rybalkin Date: Thu, 16 Mar 2023 07:51:21 +0000 Subject: [PATCH 02/42] simplify unix socket permission format (#3014) ### Pull Request Checklist * [x] I have added Go unit tests or [Complement integration tests](https://github.com/matrix-org/complement) for this PR _or_ I have justified why this PR doesn't need tests * [x] Pull request includes a [sign off below using a legally identifiable name](https://matrix-org.github.io/dendrite/development/contributing#sign-off) _or_ I have already signed off privately Signed-off-by: `Boris Rybalkin ` --- cmd/dendrite/main.go | 11 +++++++---- setup/base/base_test.go | 2 +- setup/config/config_address.go | 9 +++++++-- setup/config/config_address_test.go | 20 +++++++++++++++++++- 4 files changed, 34 insertions(+), 8 deletions(-) diff --git a/cmd/dendrite/main.go b/cmd/dendrite/main.go index 1ae348cfaf..472b8be18d 100644 --- a/cmd/dendrite/main.go +++ b/cmd/dendrite/main.go @@ -16,7 +16,6 @@ package main import ( "flag" - "io/fs" "github.com/sirupsen/logrus" @@ -34,8 +33,8 @@ var ( unixSocket = flag.String("unix-socket", "", "EXPERIMENTAL(unstable): The HTTP listening unix socket for the server (disables http[s]-bind-address feature)", ) - unixSocketPermission = flag.Int("unix-socket-permission", 0755, - "EXPERIMENTAL(unstable): The HTTP listening unix socket permission for the server", + unixSocketPermission = flag.String("unix-socket-permission", "755", + "EXPERIMENTAL(unstable): The HTTP listening unix socket permission for the server (in chmod format like 755)", ) httpBindAddr = flag.String("http-bind-address", ":8008", "The HTTP listening port for the server") httpsBindAddr = flag.String("https-bind-address", ":8448", "The HTTPS listening port for the server") @@ -59,7 +58,11 @@ func main() { } httpsAddr = https } else { - httpAddr = config.UnixSocketAddress(*unixSocket, fs.FileMode(*unixSocketPermission)) + socket, err := config.UnixSocketAddress(*unixSocket, *unixSocketPermission) + if err != nil { + logrus.WithError(err).Fatalf("Failed to parse unix socket") + } + httpAddr = socket } options := []basepkg.BaseDendriteOptions{} diff --git a/setup/base/base_test.go b/setup/base/base_test.go index 658dc5b03b..aa406db2c1 100644 --- a/setup/base/base_test.go +++ b/setup/base/base_test.go @@ -76,7 +76,7 @@ func TestLandingPage_UnixSocket(t *testing.T) { tempDir := t.TempDir() socket := path.Join(tempDir, "socket") // start base with the listener and wait for it to be started - address := config.UnixSocketAddress(socket, 0755) + address, err := config.UnixSocketAddress(socket, "755") assert.NoError(t, err) go b.SetupAndServeHTTP(address, nil, nil) time.Sleep(time.Millisecond * 100) diff --git a/setup/config/config_address.go b/setup/config/config_address.go index 0e4f0296f3..a35cc3f963 100644 --- a/setup/config/config_address.go +++ b/setup/config/config_address.go @@ -3,6 +3,7 @@ package config import ( "io/fs" "net/url" + "strconv" ) const ( @@ -32,8 +33,12 @@ func (s ServerAddress) Network() string { } } -func UnixSocketAddress(path string, perm fs.FileMode) ServerAddress { - return ServerAddress{Address: path, Scheme: NetworkUnix, UnixSocketPermission: perm} +func UnixSocketAddress(path string, perm string) (ServerAddress, error) { + permission, err := strconv.ParseInt(perm, 8, 32) + if err != nil { + return ServerAddress{}, err + } + return ServerAddress{Address: path, Scheme: NetworkUnix, UnixSocketPermission: fs.FileMode(permission)}, nil } func HTTPAddress(urlAddress string) (ServerAddress, error) { diff --git a/setup/config/config_address_test.go b/setup/config/config_address_test.go index 1be484fd50..38c96ab7f1 100644 --- a/setup/config/config_address_test.go +++ b/setup/config/config_address_test.go @@ -20,6 +20,24 @@ func TestHttpAddress_ParseBad(t *testing.T) { } func TestUnixSocketAddress_Network(t *testing.T) { - address := UnixSocketAddress("/tmp", fs.FileMode(0755)) + address, err := UnixSocketAddress("/tmp", "0755") + assert.NoError(t, err) assert.Equal(t, "unix", address.Network()) } + +func TestUnixSocketAddress_Permission_LeadingZero_Ok(t *testing.T) { + address, err := UnixSocketAddress("/tmp", "0755") + assert.NoError(t, err) + assert.Equal(t, fs.FileMode(0755), address.UnixSocketPermission) +} + +func TestUnixSocketAddress_Permission_NoLeadingZero_Ok(t *testing.T) { + address, err := UnixSocketAddress("/tmp", "755") + assert.NoError(t, err) + assert.Equal(t, fs.FileMode(0755), address.UnixSocketPermission) +} + +func TestUnixSocketAddress_Permission_NonOctal_Bad(t *testing.T) { + _, err := UnixSocketAddress("/tmp", "855") + assert.Error(t, err) +} From 5579121c6f27105342a2aea05cf9a3119d73cecb Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Fri, 17 Mar 2023 12:09:45 +0100 Subject: [PATCH 03/42] Preparations for removing `BaseDendrite` (#3016) Preparations to actually remove/replace `BaseDendrite`. Quite a few changes: - SyncAPI accepts an `fulltext.Indexer` interface (fulltext is removed from `BaseDendrite`) - Caches are removed from `BaseDendrite` - Introduces a `Router` struct (likely to change) - also fixes #2903 - Introduces a `sqlutil.ConnectionManager`, which should remove `base.DatabaseConnection` later on - probably more --- appservice/appservice_test.go | 4 +- build/dendritejs-pinecone/main.go | 14 +- build/gobind-yggdrasil/monolith.go | 20 ++- clientapi/admin_test.go | 19 +- clientapi/routing/joinroom_test.go | 4 +- clientapi/routing/login_test.go | 6 +- clientapi/routing/register_test.go | 11 +- clientapi/routing/routing.go | 8 +- .../monolith/monolith.go | 25 +-- cmd/dendrite-demo-yggdrasil/main.go | 24 +-- cmd/dendrite/main.go | 10 +- cmd/resolve-state/main.go | 2 +- federationapi/federationapi.go | 2 +- federationapi/federationapi_test.go | 6 +- federationapi/queue/queue_test.go | 13 +- federationapi/routing/profile_test.go | 2 +- federationapi/routing/query_test.go | 2 +- federationapi/routing/routing.go | 6 +- federationapi/routing/send_test.go | 2 +- federationapi/storage/postgres/storage.go | 8 +- federationapi/storage/sqlite3/storage.go | 8 +- federationapi/storage/storage.go | 9 +- federationapi/storage/storage_test.go | 12 +- federationapi/storage/storage_wasm.go | 7 +- internal/caching/impl_ristretto.go | 5 + internal/fulltext/bleve.go | 17 +- internal/fulltext/bleve_test.go | 29 +-- internal/fulltext/bleve_wasm.go | 12 +- internal/httputil/routing.go | 52 ++++++ internal/httputil/routing_test.go | 38 ++++ internal/sqlutil/connection_manager.go | 54 ++++++ internal/sqlutil/connection_manager_test.go | 56 ++++++ mediaapi/mediaapi.go | 4 +- mediaapi/routing/upload_test.go | 5 +- mediaapi/storage/postgres/mediaapi.go | 5 +- mediaapi/storage/sqlite3/mediaapi.go | 5 +- mediaapi/storage/storage.go | 8 +- mediaapi/storage/storage_test.go | 4 +- mediaapi/storage/storage_wasm.go | 6 +- relayapi/relayapi.go | 7 +- relayapi/relayapi_test.go | 16 +- relayapi/storage/postgres/storage.go | 5 +- relayapi/storage/sqlite3/storage.go | 5 +- relayapi/storage/storage.go | 8 +- relayapi/storage/storage_wasm.go | 42 +++++ roomserver/internal/api.go | 6 +- roomserver/internal/helpers/helpers_test.go | 22 +-- roomserver/internal/input/input_test.go | 6 +- roomserver/roomserver.go | 6 +- roomserver/roomserver_test.go | 17 +- roomserver/storage/postgres/storage.go | 7 +- roomserver/storage/sqlite3/storage.go | 7 +- roomserver/storage/storage.go | 9 +- roomserver/storage/storage_wasm.go | 7 +- setup/base/base.go | 167 ++++-------------- setup/jetstream/nats.go | 11 +- setup/monolith.go | 5 +- setup/mscs/msc2836/msc2836.go | 6 +- setup/mscs/msc2836/msc2836_test.go | 10 +- setup/mscs/msc2836/storage.go | 15 +- setup/mscs/msc2946/msc2946.go | 8 +- setup/mscs/mscs.go | 9 +- syncapi/consumers/clientapi.go | 2 +- syncapi/consumers/roomserver.go | 2 +- syncapi/routing/routing.go | 2 +- syncapi/routing/search.go | 2 +- syncapi/storage/postgres/syncserver.go | 8 +- syncapi/storage/sqlite3/syncserver.go | 8 +- syncapi/storage/storage.go | 9 +- syncapi/storage/storage_test.go | 35 ++-- syncapi/storage/storage_wasm.go | 7 +- syncapi/syncapi.go | 23 ++- syncapi/syncapi_test.go | 52 +++--- userapi/consumers/roomserver_test.go | 7 +- userapi/internal/device_list_update_test.go | 6 +- userapi/internal/key_api_test.go | 7 +- userapi/storage/postgres/storage.go | 13 +- userapi/storage/sqlite3/storage.go | 13 +- userapi/storage/storage.go | 16 +- userapi/storage/storage_test.go | 8 +- userapi/storage/storage_wasm.go | 23 ++- userapi/userapi.go | 8 +- userapi/userapi_test.go | 7 +- userapi/util/notify_test.go | 7 +- userapi/util/phonehomestats_test.go | 2 +- 85 files changed, 722 insertions(+), 470 deletions(-) create mode 100644 internal/httputil/routing_test.go create mode 100644 internal/sqlutil/connection_manager.go create mode 100644 internal/sqlutil/connection_manager_test.go create mode 100644 relayapi/storage/storage_wasm.go diff --git a/appservice/appservice_test.go b/appservice/appservice_test.go index de9f5aaf16..ad6f1dfc9c 100644 --- a/appservice/appservice_test.go +++ b/appservice/appservice_test.go @@ -12,6 +12,7 @@ import ( "github.com/matrix-org/dendrite/appservice" "github.com/matrix-org/dendrite/appservice/api" + "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/roomserver" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/test" @@ -123,8 +124,9 @@ func TestAppserviceInternalAPI(t *testing.T) { }, } + caches := caching.NewRistrettoCache(base.Cfg.Global.Cache.EstimatedMaxSize, base.Cfg.Global.Cache.MaxAge, caching.DisableMetrics) // Create required internal APIs - rsAPI := roomserver.NewInternalAPI(base) + rsAPI := roomserver.NewInternalAPI(base, caches) usrAPI := userapi.NewInternalAPI(base, rsAPI, nil) asAPI := appservice.NewInternalAPI(base, usrAPI, rsAPI) diff --git a/build/dendritejs-pinecone/main.go b/build/dendritejs-pinecone/main.go index 44e52286ff..96f034bdf4 100644 --- a/build/dendritejs-pinecone/main.go +++ b/build/dendritejs-pinecone/main.go @@ -29,6 +29,7 @@ import ( "github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/rooms" "github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing" "github.com/matrix-org/dendrite/federationapi" + "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/roomserver" "github.com/matrix-org/dendrite/setup" @@ -192,7 +193,8 @@ func startup() { base, userAPI, rsAPI, ) rsAPI.SetAppserviceAPI(asQuery) - fedSenderAPI := federationapi.NewInternalAPI(base, federation, rsAPI, base.Caches, keyRing, true) + caches := caching.NewRistrettoCache(base.Cfg.Global.Cache.EstimatedMaxSize, base.Cfg.Global.Cache.MaxAge, caching.EnableMetrics) + fedSenderAPI := federationapi.NewInternalAPI(base, federation, rsAPI, caches, keyRing, true) rsAPI.SetFederationAPI(fedSenderAPI, keyRing) monolith := setup.Monolith{ @@ -208,15 +210,15 @@ func startup() { //ServerKeyAPI: serverKeyAPI, ExtPublicRoomsProvider: rooms.NewPineconeRoomProvider(pRouter, pSessions, fedSenderAPI, federation), } - monolith.AddAllPublicRoutes(base) + monolith.AddAllPublicRoutes(base, caches) httpRouter := mux.NewRouter().SkipClean(true).UseEncodedPath() - httpRouter.PathPrefix(httputil.PublicClientPathPrefix).Handler(base.PublicClientAPIMux) - httpRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.PublicMediaAPIMux) + httpRouter.PathPrefix(httputil.PublicClientPathPrefix).Handler(base.Routers.Client) + httpRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.Routers.Media) p2pRouter := pSessions.Protocol("matrix").HTTP().Mux() - p2pRouter.Handle(httputil.PublicFederationPathPrefix, base.PublicFederationAPIMux) - p2pRouter.Handle(httputil.PublicMediaPathPrefix, base.PublicMediaAPIMux) + p2pRouter.Handle(httputil.PublicFederationPathPrefix, base.Routers.Federation) + p2pRouter.Handle(httputil.PublicMediaPathPrefix, base.Routers.Media) // Expose the matrix APIs via fetch - for local traffic go func() { diff --git a/build/gobind-yggdrasil/monolith.go b/build/gobind-yggdrasil/monolith.go index 32af611aef..8faad1d022 100644 --- a/build/gobind-yggdrasil/monolith.go +++ b/build/gobind-yggdrasil/monolith.go @@ -19,6 +19,7 @@ import ( "github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/yggrooms" "github.com/matrix-org/dendrite/federationapi" "github.com/matrix-org/dendrite/federationapi/api" + "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/roomserver" "github.com/matrix-org/dendrite/setup" @@ -158,10 +159,11 @@ func (m *DendriteMonolith) Start() { serverKeyAPI := &signing.YggdrasilKeys{} keyRing := serverKeyAPI.KeyRing() - rsAPI := roomserver.NewInternalAPI(base) + caches := caching.NewRistrettoCache(cfg.Global.Cache.EstimatedMaxSize, cfg.Global.Cache.MaxAge, caching.DisableMetrics) + rsAPI := roomserver.NewInternalAPI(base, caches) fsAPI := federationapi.NewInternalAPI( - base, federation, rsAPI, base.Caches, keyRing, true, + base, federation, rsAPI, caches, keyRing, true, ) userAPI := userapi.NewInternalAPI(base, rsAPI, federation) @@ -187,17 +189,17 @@ func (m *DendriteMonolith) Start() { ygg, fsAPI, federation, ), } - monolith.AddAllPublicRoutes(base) + monolith.AddAllPublicRoutes(base, caches) httpRouter := mux.NewRouter() - httpRouter.PathPrefix(httputil.PublicClientPathPrefix).Handler(base.PublicClientAPIMux) - httpRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.PublicMediaAPIMux) - httpRouter.PathPrefix(httputil.DendriteAdminPathPrefix).Handler(base.DendriteAdminMux) - httpRouter.PathPrefix(httputil.SynapseAdminPathPrefix).Handler(base.SynapseAdminMux) + httpRouter.PathPrefix(httputil.PublicClientPathPrefix).Handler(base.Routers.Client) + httpRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.Routers.Media) + httpRouter.PathPrefix(httputil.DendriteAdminPathPrefix).Handler(base.Routers.DendriteAdmin) + httpRouter.PathPrefix(httputil.SynapseAdminPathPrefix).Handler(base.Routers.SynapseAdmin) yggRouter := mux.NewRouter() - yggRouter.PathPrefix(httputil.PublicFederationPathPrefix).Handler(base.PublicFederationAPIMux) - yggRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.PublicMediaAPIMux) + yggRouter.PathPrefix(httputil.PublicFederationPathPrefix).Handler(base.Routers.Federation) + yggRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.Routers.Media) // Build both ends of a HTTP multiplex. m.httpServer = &http.Server{ diff --git a/clientapi/admin_test.go b/clientapi/admin_test.go index 300d3a88a6..46e2d30377 100644 --- a/clientapi/admin_test.go +++ b/clientapi/admin_test.go @@ -8,6 +8,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/federationapi" + "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/roomserver" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/setup/config" @@ -37,7 +38,8 @@ func TestAdminResetPassword(t *testing.T) { SigningIdentity: gomatrixserverlib.SigningIdentity{ServerName: "vh1"}, }) - rsAPI := roomserver.NewInternalAPI(base) + caches := caching.NewRistrettoCache(base.Cfg.Global.Cache.EstimatedMaxSize, base.Cfg.Global.Cache.MaxAge, caching.DisableMetrics) + rsAPI := roomserver.NewInternalAPI(base, caches) // Needed for changing the password/login userAPI := userapi.NewInternalAPI(base, rsAPI, nil) // We mostly need the userAPI for this test, so nil for other APIs/caches etc. @@ -71,7 +73,7 @@ func TestAdminResetPassword(t *testing.T) { "password": password, })) rec := httptest.NewRecorder() - base.PublicClientAPIMux.ServeHTTP(rec, req) + base.Routers.Client.ServeHTTP(rec, req) if rec.Code != http.StatusOK { t.Fatalf("failed to login: %s", rec.Body.String()) } @@ -124,7 +126,7 @@ func TestAdminResetPassword(t *testing.T) { } rec := httptest.NewRecorder() - base.DendriteAdminMux.ServeHTTP(rec, req) + base.Routers.DendriteAdmin.ServeHTTP(rec, req) t.Logf("%s", rec.Body.String()) if tc.wantOK && rec.Code != http.StatusOK { t.Fatalf("expected http status %d, got %d: %s", http.StatusOK, rec.Code, rec.Body.String()) @@ -148,15 +150,16 @@ func TestPurgeRoom(t *testing.T) { test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { base, baseClose := testrig.CreateBaseDendrite(t, dbType) + caches := caching.NewRistrettoCache(base.Cfg.Global.Cache.EstimatedMaxSize, base.Cfg.Global.Cache.MaxAge, caching.DisableMetrics) defer baseClose() fedClient := base.CreateFederationClient() - rsAPI := roomserver.NewInternalAPI(base) + rsAPI := roomserver.NewInternalAPI(base, caches) userAPI := userapi.NewInternalAPI(base, rsAPI, nil) // this starts the JetStream consumers - syncapi.AddPublicRoutes(base, userAPI, rsAPI) - federationapi.NewInternalAPI(base, fedClient, rsAPI, base.Caches, nil, true) + syncapi.AddPublicRoutes(base, userAPI, rsAPI, caches) + federationapi.NewInternalAPI(base, fedClient, rsAPI, caches, nil, true) rsAPI.SetFederationAPI(nil, nil) // Create the room @@ -193,7 +196,7 @@ func TestPurgeRoom(t *testing.T) { "password": password, })) rec := httptest.NewRecorder() - base.PublicClientAPIMux.ServeHTTP(rec, req) + base.Routers.Client.ServeHTTP(rec, req) if rec.Code != http.StatusOK { t.Fatalf("failed to login: %s", rec.Body.String()) } @@ -218,7 +221,7 @@ func TestPurgeRoom(t *testing.T) { req.Header.Set("Authorization", "Bearer "+accessTokens[aliceAdmin]) rec := httptest.NewRecorder() - base.DendriteAdminMux.ServeHTTP(rec, req) + base.Routers.DendriteAdmin.ServeHTTP(rec, req) t.Logf("%s", rec.Body.String()) if tc.wantOK && rec.Code != http.StatusOK { t.Fatalf("expected http status %d, got %d: %s", http.StatusOK, rec.Code, rec.Body.String()) diff --git a/clientapi/routing/joinroom_test.go b/clientapi/routing/joinroom_test.go index 1450ef4bdb..de8f9538db 100644 --- a/clientapi/routing/joinroom_test.go +++ b/clientapi/routing/joinroom_test.go @@ -7,6 +7,7 @@ import ( "testing" "time" + "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/dendrite/appservice" @@ -27,7 +28,8 @@ func TestJoinRoomByIDOrAlias(t *testing.T) { base, baseClose := testrig.CreateBaseDendrite(t, dbType) defer baseClose() - rsAPI := roomserver.NewInternalAPI(base) + caches := caching.NewRistrettoCache(base.Cfg.Global.Cache.EstimatedMaxSize, base.Cfg.Global.Cache.MaxAge, caching.DisableMetrics) + rsAPI := roomserver.NewInternalAPI(base, caches) userAPI := userapi.NewInternalAPI(base, rsAPI, nil) asAPI := appservice.NewInternalAPI(base, userAPI, rsAPI) rsAPI.SetFederationAPI(nil, nil) // creates the rs.Inputer etc diff --git a/clientapi/routing/login_test.go b/clientapi/routing/login_test.go index b72db9d8b5..fd3d8cba97 100644 --- a/clientapi/routing/login_test.go +++ b/clientapi/routing/login_test.go @@ -9,6 +9,7 @@ import ( "testing" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" + "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/roomserver" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/gomatrixserverlib" @@ -36,7 +37,8 @@ func TestLogin(t *testing.T) { SigningIdentity: gomatrixserverlib.SigningIdentity{ServerName: "vh1"}, }) - rsAPI := roomserver.NewInternalAPI(base) + caches := caching.NewRistrettoCache(base.Cfg.Global.Cache.EstimatedMaxSize, base.Cfg.Global.Cache.MaxAge, caching.DisableMetrics) + rsAPI := roomserver.NewInternalAPI(base, caches) // Needed for /login userAPI := userapi.NewInternalAPI(base, rsAPI, nil) @@ -114,7 +116,7 @@ func TestLogin(t *testing.T) { "password": password, })) rec := httptest.NewRecorder() - base.PublicClientAPIMux.ServeHTTP(rec, req) + base.Routers.Client.ServeHTTP(rec, req) if tc.wantOK && rec.Code != http.StatusOK { t.Fatalf("failed to login: %s", rec.Body.String()) } diff --git a/clientapi/routing/register_test.go b/clientapi/routing/register_test.go index 651e3d3d6a..c06b0ae122 100644 --- a/clientapi/routing/register_test.go +++ b/clientapi/routing/register_test.go @@ -30,6 +30,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/roomserver" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/test" @@ -407,7 +408,8 @@ func Test_register(t *testing.T) { base, baseClose := testrig.CreateBaseDendrite(t, dbType) defer baseClose() - rsAPI := roomserver.NewInternalAPI(base) + caches := caching.NewRistrettoCache(base.Cfg.Global.Cache.EstimatedMaxSize, base.Cfg.Global.Cache.MaxAge, caching.DisableMetrics) + rsAPI := roomserver.NewInternalAPI(base, caches) userAPI := userapi.NewInternalAPI(base, rsAPI, nil) for _, tc := range testCases { @@ -578,7 +580,8 @@ func TestRegisterUserWithDisplayName(t *testing.T) { defer baseClose() base.Cfg.Global.ServerName = "server" - rsAPI := roomserver.NewInternalAPI(base) + caches := caching.NewRistrettoCache(base.Cfg.Global.Cache.EstimatedMaxSize, base.Cfg.Global.Cache.MaxAge, caching.DisableMetrics) + rsAPI := roomserver.NewInternalAPI(base, caches) userAPI := userapi.NewInternalAPI(base, rsAPI, nil) deviceName, deviceID := "deviceName", "deviceID" expectedDisplayName := "DisplayName" @@ -616,8 +619,8 @@ func TestRegisterAdminUsingSharedSecret(t *testing.T) { base.Cfg.Global.ServerName = "server" sharedSecret := "dendritetest" base.Cfg.ClientAPI.RegistrationSharedSecret = sharedSecret - - rsAPI := roomserver.NewInternalAPI(base) + caches := caching.NewRistrettoCache(base.Cfg.Global.Cache.EstimatedMaxSize, base.Cfg.Global.Cache.MaxAge, caching.DisableMetrics) + rsAPI := roomserver.NewInternalAPI(base, caches) userAPI := userapi.NewInternalAPI(base, rsAPI, nil) expectedDisplayName := "rabbit" diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index 028d02e972..4ef2ac92e9 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -63,10 +63,10 @@ func Setup( extRoomsProvider api.ExtraPublicRoomsProvider, mscCfg *config.MSCs, natsClient *nats.Conn, ) { - publicAPIMux := base.PublicClientAPIMux - wkMux := base.PublicWellKnownAPIMux - synapseAdminRouter := base.SynapseAdminMux - dendriteAdminRouter := base.DendriteAdminMux + publicAPIMux := base.Routers.Client + wkMux := base.Routers.WellKnown + synapseAdminRouter := base.Routers.SynapseAdmin + dendriteAdminRouter := base.Routers.DendriteAdmin if base.EnableMetrics { prometheus.MustRegister(amtRegUsers, sendEventDuration) diff --git a/cmd/dendrite-demo-pinecone/monolith/monolith.go b/cmd/dendrite-demo-pinecone/monolith/monolith.go index ea8e985c7f..5781d6571e 100644 --- a/cmd/dendrite-demo-pinecone/monolith/monolith.go +++ b/cmd/dendrite-demo-pinecone/monolith/monolith.go @@ -37,6 +37,7 @@ import ( "github.com/matrix-org/dendrite/federationapi" federationAPI "github.com/matrix-org/dendrite/federationapi/api" "github.com/matrix-org/dendrite/federationapi/producers" + "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/relayapi" relayAPI "github.com/matrix-org/dendrite/relayapi/api" @@ -134,17 +135,17 @@ func (p *P2PMonolith) SetupDendrite(cfg *config.Dendrite, port int, enableRelayi serverKeyAPI := &signing.YggdrasilKeys{} keyRing := serverKeyAPI.KeyRing() - rsComponent := roomserver.NewInternalAPI(p.BaseDendrite) - rsAPI := rsComponent + caches := caching.NewRistrettoCache(cfg.Global.Cache.EstimatedMaxSize, cfg.Global.Cache.MaxAge, enableMetrics) + rsAPI := roomserver.NewInternalAPI(p.BaseDendrite, caches) fsAPI := federationapi.NewInternalAPI( - p.BaseDendrite, federation, rsAPI, p.BaseDendrite.Caches, keyRing, true, + p.BaseDendrite, federation, rsAPI, caches, keyRing, true, ) userAPI := userapi.NewInternalAPI(p.BaseDendrite, rsAPI, federation) asAPI := appservice.NewInternalAPI(p.BaseDendrite, userAPI, rsAPI) - rsComponent.SetFederationAPI(fsAPI, keyRing) + rsAPI.SetFederationAPI(fsAPI, keyRing) userProvider := users.NewPineconeUserProvider(p.Router, p.Sessions, userAPI, federation) roomProvider := rooms.NewPineconeRoomProvider(p.Router, p.Sessions, fsAPI, federation) @@ -161,7 +162,7 @@ func (p *P2PMonolith) SetupDendrite(cfg *config.Dendrite, port int, enableRelayi Config: &p.BaseDendrite.Cfg.FederationAPI, UserAPI: userAPI, } - relayAPI := relayapi.NewRelayInternalAPI(p.BaseDendrite, federation, rsAPI, keyRing, producer, enableRelaying) + relayAPI := relayapi.NewRelayInternalAPI(p.BaseDendrite, federation, rsAPI, keyRing, producer, enableRelaying, caches) logrus.Infof("Relaying enabled: %v", relayAPI.RelayingEnabled()) p.dendrite = setup.Monolith{ @@ -178,7 +179,7 @@ func (p *P2PMonolith) SetupDendrite(cfg *config.Dendrite, port int, enableRelayi ExtPublicRoomsProvider: roomProvider, ExtUserDirectoryProvider: userProvider, } - p.dendrite.AddAllPublicRoutes(p.BaseDendrite) + p.dendrite.AddAllPublicRoutes(p.BaseDendrite, caches) p.setupHttpServers(userProvider, enableWebsockets) } @@ -247,10 +248,10 @@ func (p *P2PMonolith) Addr() string { func (p *P2PMonolith) setupHttpServers(userProvider *users.PineconeUserProvider, enableWebsockets bool) { p.httpMux = mux.NewRouter().SkipClean(true).UseEncodedPath() - p.httpMux.PathPrefix(httputil.PublicClientPathPrefix).Handler(p.BaseDendrite.PublicClientAPIMux) - p.httpMux.PathPrefix(httputil.PublicMediaPathPrefix).Handler(p.BaseDendrite.PublicMediaAPIMux) - p.httpMux.PathPrefix(httputil.DendriteAdminPathPrefix).Handler(p.BaseDendrite.DendriteAdminMux) - p.httpMux.PathPrefix(httputil.SynapseAdminPathPrefix).Handler(p.BaseDendrite.SynapseAdminMux) + p.httpMux.PathPrefix(httputil.PublicClientPathPrefix).Handler(p.BaseDendrite.Routers.Client) + p.httpMux.PathPrefix(httputil.PublicMediaPathPrefix).Handler(p.BaseDendrite.Routers.Media) + p.httpMux.PathPrefix(httputil.DendriteAdminPathPrefix).Handler(p.BaseDendrite.Routers.DendriteAdmin) + p.httpMux.PathPrefix(httputil.SynapseAdminPathPrefix).Handler(p.BaseDendrite.Routers.SynapseAdmin) if enableWebsockets { wsUpgrader := websocket.Upgrader{ @@ -283,8 +284,8 @@ func (p *P2PMonolith) setupHttpServers(userProvider *users.PineconeUserProvider, p.pineconeMux = mux.NewRouter().SkipClean(true).UseEncodedPath() p.pineconeMux.PathPrefix(users.PublicURL).HandlerFunc(userProvider.FederatedUserProfiles) - p.pineconeMux.PathPrefix(httputil.PublicFederationPathPrefix).Handler(p.BaseDendrite.PublicFederationAPIMux) - p.pineconeMux.PathPrefix(httputil.PublicMediaPathPrefix).Handler(p.BaseDendrite.PublicMediaAPIMux) + p.pineconeMux.PathPrefix(httputil.PublicFederationPathPrefix).Handler(p.BaseDendrite.Routers.Federation) + p.pineconeMux.PathPrefix(httputil.PublicMediaPathPrefix).Handler(p.BaseDendrite.Routers.Media) pHTTP := p.Sessions.Protocol(SessionProtocol).HTTP() pHTTP.Mux().Handle(users.PublicURL, p.pineconeMux) diff --git a/cmd/dendrite-demo-yggdrasil/main.go b/cmd/dendrite-demo-yggdrasil/main.go index d759c6a731..62184719a9 100644 --- a/cmd/dendrite-demo-yggdrasil/main.go +++ b/cmd/dendrite-demo-yggdrasil/main.go @@ -27,6 +27,7 @@ import ( "path/filepath" "time" + "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/gomatrixserverlib" "github.com/gorilla/mux" @@ -156,16 +157,15 @@ func main() { serverKeyAPI := &signing.YggdrasilKeys{} keyRing := serverKeyAPI.KeyRing() - rsAPI := roomserver.NewInternalAPI( - base, - ) + caches := caching.NewRistrettoCache(base.Cfg.Global.Cache.EstimatedMaxSize, base.Cfg.Global.Cache.MaxAge, caching.EnableMetrics) + rsAPI := roomserver.NewInternalAPI(base, caches) userAPI := userapi.NewInternalAPI(base, rsAPI, federation) asAPI := appservice.NewInternalAPI(base, userAPI, rsAPI) rsAPI.SetAppserviceAPI(asAPI) fsAPI := federationapi.NewInternalAPI( - base, federation, rsAPI, base.Caches, keyRing, true, + base, federation, rsAPI, caches, keyRing, true, ) rsAPI.SetFederationAPI(fsAPI, keyRing) @@ -184,21 +184,21 @@ func main() { ygg, fsAPI, federation, ), } - monolith.AddAllPublicRoutes(base) - if err := mscs.Enable(base, &monolith); err != nil { + monolith.AddAllPublicRoutes(base, caches) + if err := mscs.Enable(base, &monolith, caches); err != nil { logrus.WithError(err).Fatalf("Failed to enable MSCs") } httpRouter := mux.NewRouter().SkipClean(true).UseEncodedPath() - httpRouter.PathPrefix(httputil.PublicClientPathPrefix).Handler(base.PublicClientAPIMux) - httpRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.PublicMediaAPIMux) - httpRouter.PathPrefix(httputil.DendriteAdminPathPrefix).Handler(base.DendriteAdminMux) - httpRouter.PathPrefix(httputil.SynapseAdminPathPrefix).Handler(base.SynapseAdminMux) + httpRouter.PathPrefix(httputil.PublicClientPathPrefix).Handler(base.Routers.Client) + httpRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.Routers.Media) + httpRouter.PathPrefix(httputil.DendriteAdminPathPrefix).Handler(base.Routers.DendriteAdmin) + httpRouter.PathPrefix(httputil.SynapseAdminPathPrefix).Handler(base.Routers.SynapseAdmin) embed.Embed(httpRouter, *instancePort, "Yggdrasil Demo") yggRouter := mux.NewRouter().SkipClean(true).UseEncodedPath() - yggRouter.PathPrefix(httputil.PublicFederationPathPrefix).Handler(base.PublicFederationAPIMux) - yggRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.PublicMediaAPIMux) + yggRouter.PathPrefix(httputil.PublicFederationPathPrefix).Handler(base.Routers.Federation) + yggRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.Routers.Media) // Build both ends of a HTTP multiplex. httpServer := &http.Server{ diff --git a/cmd/dendrite/main.go b/cmd/dendrite/main.go index 472b8be18d..29290eb985 100644 --- a/cmd/dendrite/main.go +++ b/cmd/dendrite/main.go @@ -17,6 +17,7 @@ package main import ( "flag" + "github.com/matrix-org/dendrite/internal/caching" "github.com/sirupsen/logrus" "github.com/matrix-org/dendrite/appservice" @@ -72,10 +73,11 @@ func main() { federation := base.CreateFederationClient() - rsAPI := roomserver.NewInternalAPI(base) + caches := caching.NewRistrettoCache(base.Cfg.Global.Cache.EstimatedMaxSize, base.Cfg.Global.Cache.MaxAge, caching.EnableMetrics) + rsAPI := roomserver.NewInternalAPI(base, caches) fsAPI := federationapi.NewInternalAPI( - base, federation, rsAPI, base.Caches, nil, false, + base, federation, rsAPI, caches, nil, false, ) keyRing := fsAPI.KeyRing() @@ -104,10 +106,10 @@ func main() { RoomserverAPI: rsAPI, UserAPI: userAPI, } - monolith.AddAllPublicRoutes(base) + monolith.AddAllPublicRoutes(base, caches) if len(base.Cfg.MSCs.MSCs) > 0 { - if err := mscs.Enable(base, &monolith); err != nil { + if err := mscs.Enable(base, &monolith, caches); err != nil { logrus.WithError(err).Fatalf("Failed to enable MSCs") } } diff --git a/cmd/resolve-state/main.go b/cmd/resolve-state/main.go index a9cc80cb7c..099daaa9a3 100644 --- a/cmd/resolve-state/main.go +++ b/cmd/resolve-state/main.go @@ -55,7 +55,7 @@ func main() { fmt.Println("Fetching", len(snapshotNIDs), "snapshot NIDs") roomserverDB, err := storage.Open( - base, &cfg.RoomServer.Database, + base.ProcessContext.Context(), base.ConnectionManager, &cfg.RoomServer.Database, caching.NewRistrettoCache(128*1024*1024, time.Hour, true), ) if err != nil { diff --git a/federationapi/federationapi.go b/federationapi/federationapi.go index ec482659a1..8a4237bae5 100644 --- a/federationapi/federationapi.go +++ b/federationapi/federationapi.go @@ -94,7 +94,7 @@ func NewInternalAPI( ) api.FederationInternalAPI { cfg := &base.Cfg.FederationAPI - federationDB, err := storage.NewDatabase(base, &cfg.Database, base.Caches, base.Cfg.Global.IsLocalServerName) + federationDB, err := storage.NewDatabase(base.ProcessContext.Context(), base.ConnectionManager, &cfg.Database, caches, base.Cfg.Global.IsLocalServerName) if err != nil { logrus.WithError(err).Panic("failed to connect to federation sender db") } diff --git a/federationapi/federationapi_test.go b/federationapi/federationapi_test.go index 57d4b96446..8aea96a731 100644 --- a/federationapi/federationapi_test.go +++ b/federationapi/federationapi_test.go @@ -10,6 +10,7 @@ import ( "testing" "time" + "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/gomatrix" "github.com/matrix-org/gomatrixserverlib" "github.com/nats-io/nats.go" @@ -163,6 +164,7 @@ func TestFederationAPIJoinThenKeyUpdate(t *testing.T) { func testFederationAPIJoinThenKeyUpdate(t *testing.T, dbType test.DBType) { base, close := testrig.CreateBaseDendrite(t, dbType) + caches := caching.NewRistrettoCache(base.Cfg.Global.Cache.EstimatedMaxSize, base.Cfg.Global.Cache.MaxAge, caching.DisableMetrics) base.Cfg.FederationAPI.PreferDirectFetch = true base.Cfg.FederationAPI.KeyPerspectives = nil defer close() @@ -212,7 +214,7 @@ func testFederationAPIJoinThenKeyUpdate(t *testing.T, dbType test.DBType) { }, }, } - fsapi := federationapi.NewInternalAPI(base, fc, rsapi, base.Caches, nil, false) + fsapi := federationapi.NewInternalAPI(base, fc, rsapi, caches, nil, false) var resp api.PerformJoinResponse fsapi.PerformJoin(context.Background(), &api.PerformJoinRequest{ @@ -278,7 +280,7 @@ func TestRoomsV3URLEscapeDoNot404(t *testing.T) { // TODO: This is pretty fragile, as if anything calls anything on these nils this test will break. // Unfortunately, it makes little sense to instantiate these dependencies when we just want to test routing. federationapi.AddPublicRoutes(b, nil, nil, keyRing, nil, &internal.FederationInternalAPI{}, nil) - baseURL, cancel := test.ListenAndServe(t, b.PublicFederationAPIMux, true) + baseURL, cancel := test.ListenAndServe(t, b.Routers.Federation, true) defer cancel() serverName := gomatrixserverlib.ServerName(strings.TrimPrefix(baseURL, "https://")) diff --git a/federationapi/queue/queue_test.go b/federationapi/queue/queue_test.go index bccfb34289..c3fb10561c 100644 --- a/federationapi/queue/queue_test.go +++ b/federationapi/queue/queue_test.go @@ -21,6 +21,8 @@ import ( "testing" "time" + "github.com/matrix-org/dendrite/internal/caching" + "github.com/matrix-org/dendrite/test/testrig" "go.uber.org/atomic" "gotest.tools/v3/poll" @@ -34,17 +36,17 @@ import ( "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/process" "github.com/matrix-org/dendrite/test" - "github.com/matrix-org/dendrite/test/testrig" ) func mustCreateFederationDatabase(t *testing.T, dbType test.DBType, realDatabase bool) (storage.Database, *process.ProcessContext, func()) { if realDatabase { // Real Database/s b, baseClose := testrig.CreateBaseDendrite(t, dbType) + caches := caching.NewRistrettoCache(b.Cfg.Global.Cache.EstimatedMaxSize, b.Cfg.Global.Cache.MaxAge, caching.DisableMetrics) connStr, dbClose := test.PrepareDBConnectionString(t, dbType) - db, err := storage.NewDatabase(b, &config.DatabaseOptions{ + db, err := storage.NewDatabase(b.ProcessContext.Context(), b.ConnectionManager, &config.DatabaseOptions{ ConnectionString: config.DataSource(connStr), - }, b.Caches, b.Cfg.Global.IsLocalServerName) + }, caches, b.Cfg.Global.IsLocalServerName) if err != nil { t.Fatalf("NewDatabase returned %s", err) } @@ -55,10 +57,7 @@ func mustCreateFederationDatabase(t *testing.T, dbType test.DBType, realDatabase } else { // Fake Database db := test.NewInMemoryFederationDatabase() - b := struct { - ProcessContext *process.ProcessContext - }{ProcessContext: process.NewProcessContext()} - return db, b.ProcessContext, func() {} + return db, process.NewProcessContext(), func() {} } } diff --git a/federationapi/routing/profile_test.go b/federationapi/routing/profile_test.go index 3b9d576bf7..df494a7432 100644 --- a/federationapi/routing/profile_test.go +++ b/federationapi/routing/profile_test.go @@ -50,7 +50,7 @@ func TestHandleQueryProfile(t *testing.T) { defer close() fedMux := mux.NewRouter().SkipClean(true).PathPrefix(httputil.PublicFederationPathPrefix).Subrouter().UseEncodedPath() - base.PublicFederationAPIMux = fedMux + base.Routers.Federation = fedMux base.Cfg.FederationAPI.Matrix.SigningIdentity.ServerName = testOrigin base.Cfg.FederationAPI.Matrix.Metrics.Enabled = false fedClient := fakeFedClient{} diff --git a/federationapi/routing/query_test.go b/federationapi/routing/query_test.go index d839a16b87..69cf7047d6 100644 --- a/federationapi/routing/query_test.go +++ b/federationapi/routing/query_test.go @@ -50,7 +50,7 @@ func TestHandleQueryDirectory(t *testing.T) { defer close() fedMux := mux.NewRouter().SkipClean(true).PathPrefix(httputil.PublicFederationPathPrefix).Subrouter().UseEncodedPath() - base.PublicFederationAPIMux = fedMux + base.Routers.Federation = fedMux base.Cfg.FederationAPI.Matrix.SigningIdentity.ServerName = testOrigin base.Cfg.FederationAPI.Matrix.Metrics.Enabled = false fedClient := fakeFedClient{} diff --git a/federationapi/routing/routing.go b/federationapi/routing/routing.go index 324740ddca..c86d18d2f9 100644 --- a/federationapi/routing/routing.go +++ b/federationapi/routing/routing.go @@ -65,9 +65,9 @@ func Setup( servers federationAPI.ServersInRoomProvider, producer *producers.SyncAPIProducer, ) { - fedMux := base.PublicFederationAPIMux - keyMux := base.PublicKeyAPIMux - wkMux := base.PublicWellKnownAPIMux + fedMux := base.Routers.Federation + keyMux := base.Routers.Keys + wkMux := base.Routers.WellKnown cfg := &base.Cfg.FederationAPI if base.EnableMetrics { diff --git a/federationapi/routing/send_test.go b/federationapi/routing/send_test.go index eed4e7e69f..53e1399ab3 100644 --- a/federationapi/routing/send_test.go +++ b/federationapi/routing/send_test.go @@ -48,7 +48,7 @@ func TestHandleSend(t *testing.T) { defer close() fedMux := mux.NewRouter().SkipClean(true).PathPrefix(httputil.PublicFederationPathPrefix).Subrouter().UseEncodedPath() - base.PublicFederationAPIMux = fedMux + base.Routers.Federation = fedMux base.Cfg.FederationAPI.Matrix.SigningIdentity.ServerName = testOrigin base.Cfg.FederationAPI.Matrix.Metrics.Enabled = false fedapi := fedAPI.NewInternalAPI(base, nil, nil, nil, nil, true) diff --git a/federationapi/storage/postgres/storage.go b/federationapi/storage/postgres/storage.go index b81f128e7b..468567cf09 100644 --- a/federationapi/storage/postgres/storage.go +++ b/federationapi/storage/postgres/storage.go @@ -16,6 +16,7 @@ package postgres import ( + "context" "database/sql" "fmt" @@ -23,7 +24,6 @@ import ( "github.com/matrix-org/dendrite/federationapi/storage/shared" "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/internal/sqlutil" - "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/gomatrixserverlib" ) @@ -36,10 +36,10 @@ type Database struct { } // NewDatabase opens a new database -func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions, cache caching.FederationCache, isLocalServerName func(gomatrixserverlib.ServerName) bool) (*Database, error) { +func NewDatabase(ctx context.Context, conMan sqlutil.Connections, dbProperties *config.DatabaseOptions, cache caching.FederationCache, isLocalServerName func(gomatrixserverlib.ServerName) bool) (*Database, error) { var d Database var err error - if d.db, d.writer, err = base.DatabaseConnection(dbProperties, sqlutil.NewDummyWriter()); err != nil { + if d.db, d.writer, err = conMan.Connection(dbProperties); err != nil { return nil, err } blacklist, err := NewPostgresBlacklistTable(d.db) @@ -95,7 +95,7 @@ func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions, Version: "federationsender: drop federationsender_rooms", Up: deltas.UpRemoveRoomsTable, }) - err = m.Up(base.Context()) + err = m.Up(ctx) if err != nil { return nil, err } diff --git a/federationapi/storage/sqlite3/storage.go b/federationapi/storage/sqlite3/storage.go index 1e7e41a2c2..c64c9a4f02 100644 --- a/federationapi/storage/sqlite3/storage.go +++ b/federationapi/storage/sqlite3/storage.go @@ -15,13 +15,13 @@ package sqlite3 import ( + "context" "database/sql" "github.com/matrix-org/dendrite/federationapi/storage/shared" "github.com/matrix-org/dendrite/federationapi/storage/sqlite3/deltas" "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/internal/sqlutil" - "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/gomatrixserverlib" ) @@ -34,10 +34,10 @@ type Database struct { } // NewDatabase opens a new database -func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions, cache caching.FederationCache, isLocalServerName func(gomatrixserverlib.ServerName) bool) (*Database, error) { +func NewDatabase(ctx context.Context, conMan sqlutil.Connections, dbProperties *config.DatabaseOptions, cache caching.FederationCache, isLocalServerName func(gomatrixserverlib.ServerName) bool) (*Database, error) { var d Database var err error - if d.db, d.writer, err = base.DatabaseConnection(dbProperties, sqlutil.NewExclusiveWriter()); err != nil { + if d.db, d.writer, err = conMan.Connection(dbProperties); err != nil { return nil, err } blacklist, err := NewSQLiteBlacklistTable(d.db) @@ -93,7 +93,7 @@ func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions, Version: "federationsender: drop federationsender_rooms", Up: deltas.UpRemoveRoomsTable, }) - err = m.Up(base.Context()) + err = m.Up(ctx) if err != nil { return nil, err } diff --git a/federationapi/storage/storage.go b/federationapi/storage/storage.go index 142e281ea1..4eb9d2c984 100644 --- a/federationapi/storage/storage.go +++ b/federationapi/storage/storage.go @@ -18,23 +18,24 @@ package storage import ( + "context" "fmt" "github.com/matrix-org/dendrite/federationapi/storage/postgres" "github.com/matrix-org/dendrite/federationapi/storage/sqlite3" "github.com/matrix-org/dendrite/internal/caching" - "github.com/matrix-org/dendrite/setup/base" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/gomatrixserverlib" ) // NewDatabase opens a new database -func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions, cache caching.FederationCache, isLocalServerName func(gomatrixserverlib.ServerName) bool) (Database, error) { +func NewDatabase(ctx context.Context, conMan sqlutil.Connections, dbProperties *config.DatabaseOptions, cache caching.FederationCache, isLocalServerName func(gomatrixserverlib.ServerName) bool) (Database, error) { switch { case dbProperties.ConnectionString.IsSQLite(): - return sqlite3.NewDatabase(base, dbProperties, cache, isLocalServerName) + return sqlite3.NewDatabase(ctx, conMan, dbProperties, cache, isLocalServerName) case dbProperties.ConnectionString.IsPostgres(): - return postgres.NewDatabase(base, dbProperties, cache, isLocalServerName) + return postgres.NewDatabase(ctx, conMan, dbProperties, cache, isLocalServerName) default: return nil, fmt.Errorf("unexpected database type") } diff --git a/federationapi/storage/storage_test.go b/federationapi/storage/storage_test.go index 1d2a13e81d..a02b71b60c 100644 --- a/federationapi/storage/storage_test.go +++ b/federationapi/storage/storage_test.go @@ -7,26 +7,28 @@ import ( "time" "github.com/matrix-org/dendrite/federationapi/storage" + "github.com/matrix-org/dendrite/internal/caching" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/test" - "github.com/matrix-org/dendrite/test/testrig" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" "github.com/stretchr/testify/assert" ) func mustCreateFederationDatabase(t *testing.T, dbType test.DBType) (storage.Database, func()) { - b, baseClose := testrig.CreateBaseDendrite(t, dbType) + caches := caching.NewRistrettoCache(8*1024*1024, time.Hour, false) connStr, dbClose := test.PrepareDBConnectionString(t, dbType) - db, err := storage.NewDatabase(b, &config.DatabaseOptions{ + ctx := context.Background() + cm := sqlutil.NewConnectionManager() + db, err := storage.NewDatabase(ctx, cm, &config.DatabaseOptions{ ConnectionString: config.DataSource(connStr), - }, b.Caches, func(server gomatrixserverlib.ServerName) bool { return server == "localhost" }) + }, caches, func(server gomatrixserverlib.ServerName) bool { return server == "localhost" }) if err != nil { t.Fatalf("NewDatabase returned %s", err) } return db, func() { dbClose() - baseClose() } } diff --git a/federationapi/storage/storage_wasm.go b/federationapi/storage/storage_wasm.go index 84d5a3a4c6..d1652d7125 100644 --- a/federationapi/storage/storage_wasm.go +++ b/federationapi/storage/storage_wasm.go @@ -15,20 +15,21 @@ package storage import ( + "context" "fmt" "github.com/matrix-org/dendrite/federationapi/storage/sqlite3" "github.com/matrix-org/dendrite/internal/caching" - "github.com/matrix-org/dendrite/setup/base" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/gomatrixserverlib" ) // NewDatabase opens a new database -func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions, cache caching.FederationCache, serverName gomatrixserverlib.ServerName) (Database, error) { +func NewDatabase(ctx context.Context, conMan sqlutil.Connections, dbProperties *config.DatabaseOptions, cache caching.FederationCache, isLocalServerName func(gomatrixserverlib.ServerName) bool) (Database, error) { switch { case dbProperties.ConnectionString.IsSQLite(): - return sqlite3.NewDatabase(base, dbProperties, cache, serverName) + return sqlite3.NewDatabase(ctx, conMan, dbProperties, cache, isLocalServerName) case dbProperties.ConnectionString.IsPostgres(): return nil, fmt.Errorf("can't use Postgres implementation") default: diff --git a/internal/caching/impl_ristretto.go b/internal/caching/impl_ristretto.go index 106b9c99fb..7663ddcb95 100644 --- a/internal/caching/impl_ristretto.go +++ b/internal/caching/impl_ristretto.go @@ -46,6 +46,11 @@ const ( eventStateKeyNIDCache ) +const ( + DisableMetrics = false + EnableMetrics = true +) + func NewRistrettoCache(maxCost config.DataUnit, maxAge time.Duration, enablePrometheus bool) *Caches { cache, err := ristretto.NewCache(&ristretto.Config{ NumCounters: int64((maxCost / 1024) * 10), // 10 counters per 1KB data, affects bloom filter size diff --git a/internal/fulltext/bleve.go b/internal/fulltext/bleve.go index 7187861dd3..08cef8a97f 100644 --- a/internal/fulltext/bleve.go +++ b/internal/fulltext/bleve.go @@ -18,10 +18,10 @@ package fulltext import ( + "context" "strings" "github.com/blevesearch/bleve/v2" - // side effect imports to allow all possible languages _ "github.com/blevesearch/bleve/v2/analysis/lang/ar" _ "github.com/blevesearch/bleve/v2/analysis/lang/cjk" @@ -55,6 +55,13 @@ type Search struct { FulltextIndex bleve.Index } +type Indexer interface { + Index(elements ...IndexElement) error + Delete(eventID string) error + Search(term string, roomIDs, keys []string, limit, from int, orderByStreamPos bool) (*bleve.SearchResult, error) + Close() error +} + // IndexElement describes the layout of an element to index type IndexElement struct { EventID string @@ -77,12 +84,18 @@ func (i *IndexElement) SetContentType(v string) { } // New opens a new/existing fulltext index -func New(cfg config.Fulltext) (fts *Search, err error) { +func New(ctx context.Context, cfg config.Fulltext) (fts *Search, err error) { fts = &Search{} fts.FulltextIndex, err = openIndex(cfg) if err != nil { return nil, err } + go func() { + // Wait for the context (should be from process.ProcessContext) to be + // done, indicating that Dendrite is shutting down. + <-ctx.Done() + _ = fts.Close() + }() return fts, nil } diff --git a/internal/fulltext/bleve_test.go b/internal/fulltext/bleve_test.go index d16397a45a..6ce6914548 100644 --- a/internal/fulltext/bleve_test.go +++ b/internal/fulltext/bleve_test.go @@ -18,6 +18,7 @@ import ( "reflect" "testing" + "github.com/matrix-org/dendrite/setup/process" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" @@ -25,7 +26,7 @@ import ( "github.com/matrix-org/dendrite/setup/config" ) -func mustOpenIndex(t *testing.T, tempDir string) *fulltext.Search { +func mustOpenIndex(t *testing.T, tempDir string) (*fulltext.Search, *process.ProcessContext) { t.Helper() cfg := config.Fulltext{ Enabled: true, @@ -36,11 +37,12 @@ func mustOpenIndex(t *testing.T, tempDir string) *fulltext.Search { cfg.IndexPath = config.Path(tempDir) cfg.InMemory = false } - fts, err := fulltext.New(cfg) + ctx := process.NewProcessContext() + fts, err := fulltext.New(ctx.Context(), cfg) if err != nil { t.Fatal("failed to open fulltext index:", err) } - return fts + return fts, ctx } func mustAddTestData(t *testing.T, fts *fulltext.Search, firstStreamPos int64) (eventIDs, roomIDs []string) { @@ -93,19 +95,17 @@ func mustAddTestData(t *testing.T, fts *fulltext.Search, firstStreamPos int64) ( func TestOpen(t *testing.T) { dataDir := t.TempDir() - fts := mustOpenIndex(t, dataDir) - if err := fts.Close(); err != nil { - t.Fatal("unable to close fulltext index", err) - } + _, ctx := mustOpenIndex(t, dataDir) + ctx.ShutdownDendrite() // open existing index - fts = mustOpenIndex(t, dataDir) - defer fts.Close() + _, ctx = mustOpenIndex(t, dataDir) + ctx.ShutdownDendrite() } func TestIndex(t *testing.T) { - fts := mustOpenIndex(t, "") - defer fts.Close() + fts, ctx := mustOpenIndex(t, "") + defer ctx.ShutdownDendrite() // add some data var streamPos int64 = 1 @@ -128,8 +128,8 @@ func TestIndex(t *testing.T) { } func TestDelete(t *testing.T) { - fts := mustOpenIndex(t, "") - defer fts.Close() + fts, ctx := mustOpenIndex(t, "") + defer ctx.ShutdownDendrite() eventIDs, roomIDs := mustAddTestData(t, fts, 0) res1, err := fts.Search("lorem", roomIDs[:1], nil, 50, 0, false) if err != nil { @@ -224,7 +224,8 @@ func TestSearch(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - f := mustOpenIndex(t, "") + f, ctx := mustOpenIndex(t, "") + defer ctx.ShutdownDendrite() eventIDs, roomIDs := mustAddTestData(t, f, 0) var searchRooms []string for _, x := range tt.args.roomIndex { diff --git a/internal/fulltext/bleve_wasm.go b/internal/fulltext/bleve_wasm.go index a69a8926ec..0053ed8c27 100644 --- a/internal/fulltext/bleve_wasm.go +++ b/internal/fulltext/bleve_wasm.go @@ -15,8 +15,9 @@ package fulltext import ( - "github.com/matrix-org/dendrite/setup/config" "time" + + "github.com/matrix-org/dendrite/setup/config" ) type Search struct{} @@ -28,6 +29,13 @@ type IndexElement struct { StreamPosition int64 } +type Indexer interface { + Index(elements ...IndexElement) error + Delete(eventID string) error + Search(term string, roomIDs, keys []string, limit, from int, orderByStreamPos bool) (SearchResult, error) + Close() error +} + type SearchResult struct { Status interface{} `json:"status"` Request *interface{} `json:"request"` @@ -48,7 +56,7 @@ func (f *Search) Close() error { return nil } -func (f *Search) Index(e IndexElement) error { +func (f *Search) Index(e ...IndexElement) error { return nil } diff --git a/internal/httputil/routing.go b/internal/httputil/routing.go index 0bd3655ecd..c733c8ce7b 100644 --- a/internal/httputil/routing.go +++ b/internal/httputil/routing.go @@ -15,7 +15,10 @@ package httputil import ( + "net/http" "net/url" + + "github.com/gorilla/mux" ) // URLDecodeMapValues is a function that iterates through each of the items in a @@ -33,3 +36,52 @@ func URLDecodeMapValues(vmap map[string]string) (map[string]string, error) { return decoded, nil } + +type Routers struct { + Client *mux.Router + Federation *mux.Router + Keys *mux.Router + Media *mux.Router + WellKnown *mux.Router + Static *mux.Router + DendriteAdmin *mux.Router + SynapseAdmin *mux.Router +} + +func NewRouters() Routers { + r := Routers{ + Client: mux.NewRouter().SkipClean(true).PathPrefix(PublicClientPathPrefix).Subrouter().UseEncodedPath(), + Federation: mux.NewRouter().SkipClean(true).PathPrefix(PublicFederationPathPrefix).Subrouter().UseEncodedPath(), + Keys: mux.NewRouter().SkipClean(true).PathPrefix(PublicKeyPathPrefix).Subrouter().UseEncodedPath(), + Media: mux.NewRouter().SkipClean(true).PathPrefix(PublicMediaPathPrefix).Subrouter().UseEncodedPath(), + WellKnown: mux.NewRouter().SkipClean(true).PathPrefix(PublicWellKnownPrefix).Subrouter().UseEncodedPath(), + Static: mux.NewRouter().SkipClean(true).PathPrefix(PublicStaticPath).Subrouter().UseEncodedPath(), + DendriteAdmin: mux.NewRouter().SkipClean(true).PathPrefix(DendriteAdminPathPrefix).Subrouter().UseEncodedPath(), + SynapseAdmin: mux.NewRouter().SkipClean(true).PathPrefix(SynapseAdminPathPrefix).Subrouter().UseEncodedPath(), + } + r.configureHTTPErrors() + return r +} + +var NotAllowedHandler = WrapHandlerInCORS(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusMethodNotAllowed) + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"errcode":"M_UNRECOGNIZED","error":"Unrecognized request"}`)) // nolint:misspell +})) + +var NotFoundCORSHandler = WrapHandlerInCORS(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"errcode":"M_UNRECOGNIZED","error":"Unrecognized request"}`)) // nolint:misspell +})) + +func (r *Routers) configureHTTPErrors() { + for _, router := range []*mux.Router{ + r.Client, r.Federation, r.Keys, + r.Media, r.WellKnown, r.Static, + r.DendriteAdmin, r.SynapseAdmin, + } { + router.NotFoundHandler = NotFoundCORSHandler + router.MethodNotAllowedHandler = NotAllowedHandler + } +} diff --git a/internal/httputil/routing_test.go b/internal/httputil/routing_test.go new file mode 100644 index 0000000000..21e2bf48ad --- /dev/null +++ b/internal/httputil/routing_test.go @@ -0,0 +1,38 @@ +package httputil + +import ( + "net/http" + "net/http/httptest" + "path/filepath" + "testing" +) + +func TestRoutersError(t *testing.T) { + r := NewRouters() + + // not found test + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, filepath.Join(PublicFederationPathPrefix, "doesnotexist"), nil) + r.Federation.ServeHTTP(rec, req) + if rec.Code != http.StatusNotFound { + t.Fatalf("unexpected status code: %d - %s", rec.Code, rec.Body.String()) + } + if ct := rec.Header().Get("Content-Type"); ct != "application/json" { + t.Fatalf("unexpected content-type: %s", ct) + } + + // not allowed test + r.DendriteAdmin. + Handle("/test", http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {})). + Methods(http.MethodPost) + + rec = httptest.NewRecorder() + req = httptest.NewRequest(http.MethodGet, filepath.Join(DendriteAdminPathPrefix, "test"), nil) + r.DendriteAdmin.ServeHTTP(rec, req) + if rec.Code != http.StatusMethodNotAllowed { + t.Fatalf("unexpected status code: %d - %s", rec.Code, rec.Body.String()) + } + if ct := rec.Header().Get("Content-Type"); ct != "application/json" { + t.Fatalf("unexpected content-type: %s", ct) + } +} diff --git a/internal/sqlutil/connection_manager.go b/internal/sqlutil/connection_manager.go new file mode 100644 index 0000000000..cefd9f8081 --- /dev/null +++ b/internal/sqlutil/connection_manager.go @@ -0,0 +1,54 @@ +// Copyright 2023 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sqlutil + +import ( + "database/sql" + "fmt" + + "github.com/matrix-org/dendrite/setup/config" +) + +type Connections struct { + db *sql.DB + writer Writer +} + +func NewConnectionManager() Connections { + return Connections{} +} + +func (c *Connections) Connection(dbProperties *config.DatabaseOptions) (*sql.DB, Writer, error) { + writer := NewDummyWriter() + if dbProperties.ConnectionString.IsSQLite() { + writer = NewExclusiveWriter() + } + if dbProperties.ConnectionString != "" || c.db == nil { + var err error + // Open a new database connection using the supplied config. + c.db, err = Open(dbProperties, writer) + if err != nil { + return nil, nil, err + } + c.writer = writer + return c.db, c.writer, nil + } + if c.db != nil && c.writer != nil { + // Ignore the supplied config and return the global pool and + // writer. + return c.db, c.writer, nil + } + return nil, nil, fmt.Errorf("no database connections configured") +} diff --git a/internal/sqlutil/connection_manager_test.go b/internal/sqlutil/connection_manager_test.go new file mode 100644 index 0000000000..610629d5ea --- /dev/null +++ b/internal/sqlutil/connection_manager_test.go @@ -0,0 +1,56 @@ +package sqlutil_test + +import ( + "reflect" + "testing" + + "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/dendrite/test" +) + +func TestConnectionManager(t *testing.T) { + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + conStr, close := test.PrepareDBConnectionString(t, dbType) + t.Cleanup(close) + cm := sqlutil.NewConnectionManager() + + dbProps := &config.DatabaseOptions{ConnectionString: config.DataSource(string(conStr))} + db, writer, err := cm.Connection(dbProps) + if err != nil { + t.Fatal(err) + } + + switch dbType { + case test.DBTypeSQLite: + _, ok := writer.(*sqlutil.ExclusiveWriter) + if !ok { + t.Fatalf("expected exclusive writer") + } + case test.DBTypePostgres: + _, ok := writer.(*sqlutil.DummyWriter) + if !ok { + t.Fatalf("expected dummy writer") + } + } + + // test global db pool + dbGlobal, writerGlobal, err := cm.Connection(&config.DatabaseOptions{}) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(db, dbGlobal) { + t.Fatalf("expected database connection to be reused") + } + if !reflect.DeepEqual(writer, writerGlobal) { + t.Fatalf("expected database writer to be reused") + } + + // test invalid connection string configured + cm = sqlutil.NewConnectionManager() + _, _, err = cm.Connection(&config.DatabaseOptions{ConnectionString: "http://"}) + if err == nil { + t.Fatal("expected an error but got none") + } + }) +} diff --git a/mediaapi/mediaapi.go b/mediaapi/mediaapi.go index 4792c996d9..42e0ea88fb 100644 --- a/mediaapi/mediaapi.go +++ b/mediaapi/mediaapi.go @@ -32,12 +32,12 @@ func AddPublicRoutes( cfg := &base.Cfg.MediaAPI rateCfg := &base.Cfg.ClientAPI.RateLimiting - mediaDB, err := storage.NewMediaAPIDatasource(base, &cfg.Database) + mediaDB, err := storage.NewMediaAPIDatasource(base.ConnectionManager, &cfg.Database) if err != nil { logrus.WithError(err).Panicf("failed to connect to media db") } routing.Setup( - base.PublicMediaAPIMux, cfg, rateCfg, mediaDB, userAPI, client, + base.Routers.Media, cfg, rateCfg, mediaDB, userAPI, client, ) } diff --git a/mediaapi/routing/upload_test.go b/mediaapi/routing/upload_test.go index 420d0eba9c..d4fb45d1b4 100644 --- a/mediaapi/routing/upload_test.go +++ b/mediaapi/routing/upload_test.go @@ -9,6 +9,7 @@ import ( "strings" "testing" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/mediaapi/fileutils" "github.com/matrix-org/dendrite/mediaapi/storage" "github.com/matrix-org/dendrite/mediaapi/types" @@ -49,8 +50,8 @@ func Test_uploadRequest_doUpload(t *testing.T) { // create testdata folder and remove when done _ = os.Mkdir(testdataPath, os.ModePerm) defer fileutils.RemoveDir(types.Path(testdataPath), nil) - - db, err := storage.NewMediaAPIDatasource(nil, &config.DatabaseOptions{ + cm := sqlutil.NewConnectionManager() + db, err := storage.NewMediaAPIDatasource(cm, &config.DatabaseOptions{ ConnectionString: "file::memory:?cache=shared", MaxOpenConnections: 100, MaxIdleConnections: 2, diff --git a/mediaapi/storage/postgres/mediaapi.go b/mediaapi/storage/postgres/mediaapi.go index 30ec64f84e..5b66877435 100644 --- a/mediaapi/storage/postgres/mediaapi.go +++ b/mediaapi/storage/postgres/mediaapi.go @@ -20,13 +20,12 @@ import ( _ "github.com/lib/pq" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/mediaapi/storage/shared" - "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/config" ) // NewDatabase opens a postgres database. -func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions) (*shared.Database, error) { - db, writer, err := base.DatabaseConnection(dbProperties, sqlutil.NewDummyWriter()) +func NewDatabase(conMan sqlutil.Connections, dbProperties *config.DatabaseOptions) (*shared.Database, error) { + db, writer, err := conMan.Connection(dbProperties) if err != nil { return nil, err } diff --git a/mediaapi/storage/sqlite3/mediaapi.go b/mediaapi/storage/sqlite3/mediaapi.go index c0ab10e9fe..4d484f326c 100644 --- a/mediaapi/storage/sqlite3/mediaapi.go +++ b/mediaapi/storage/sqlite3/mediaapi.go @@ -19,13 +19,12 @@ import ( // Import the postgres database driver. "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/mediaapi/storage/shared" - "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/config" ) // NewDatabase opens a SQLIte database. -func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions) (*shared.Database, error) { - db, writer, err := base.DatabaseConnection(dbProperties, sqlutil.NewExclusiveWriter()) +func NewDatabase(conMan sqlutil.Connections, dbProperties *config.DatabaseOptions) (*shared.Database, error) { + db, writer, err := conMan.Connection(dbProperties) if err != nil { return nil, err } diff --git a/mediaapi/storage/storage.go b/mediaapi/storage/storage.go index f673ae7e6a..8e67af9f9d 100644 --- a/mediaapi/storage/storage.go +++ b/mediaapi/storage/storage.go @@ -20,19 +20,19 @@ package storage import ( "fmt" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/mediaapi/storage/postgres" "github.com/matrix-org/dendrite/mediaapi/storage/sqlite3" - "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/config" ) // NewMediaAPIDatasource opens a database connection. -func NewMediaAPIDatasource(base *base.BaseDendrite, dbProperties *config.DatabaseOptions) (Database, error) { +func NewMediaAPIDatasource(conMan sqlutil.Connections, dbProperties *config.DatabaseOptions) (Database, error) { switch { case dbProperties.ConnectionString.IsSQLite(): - return sqlite3.NewDatabase(base, dbProperties) + return sqlite3.NewDatabase(conMan, dbProperties) case dbProperties.ConnectionString.IsPostgres(): - return postgres.NewDatabase(base, dbProperties) + return postgres.NewDatabase(conMan, dbProperties) default: return nil, fmt.Errorf("unexpected database type") } diff --git a/mediaapi/storage/storage_test.go b/mediaapi/storage/storage_test.go index 81f0a5d24d..11febd2756 100644 --- a/mediaapi/storage/storage_test.go +++ b/mediaapi/storage/storage_test.go @@ -5,6 +5,7 @@ import ( "reflect" "testing" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/mediaapi/storage" "github.com/matrix-org/dendrite/mediaapi/types" "github.com/matrix-org/dendrite/setup/config" @@ -13,7 +14,8 @@ import ( func mustCreateDatabase(t *testing.T, dbType test.DBType) (storage.Database, func()) { connStr, close := test.PrepareDBConnectionString(t, dbType) - db, err := storage.NewMediaAPIDatasource(nil, &config.DatabaseOptions{ + cm := sqlutil.NewConnectionManager() + db, err := storage.NewMediaAPIDatasource(cm, &config.DatabaseOptions{ ConnectionString: config.DataSource(connStr), }) if err != nil { diff --git a/mediaapi/storage/storage_wasm.go b/mediaapi/storage/storage_wasm.go index 41e4a28c03..47ee3792c6 100644 --- a/mediaapi/storage/storage_wasm.go +++ b/mediaapi/storage/storage_wasm.go @@ -17,16 +17,16 @@ package storage import ( "fmt" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/mediaapi/storage/sqlite3" - "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/config" ) // Open opens a postgres database. -func NewMediaAPIDatasource(base *base.BaseDendrite, dbProperties *config.DatabaseOptions) (Database, error) { +func NewMediaAPIDatasource(conMan sqlutil.Connections, dbProperties *config.DatabaseOptions) (Database, error) { switch { case dbProperties.ConnectionString.IsSQLite(): - return sqlite3.NewDatabase(base, dbProperties) + return sqlite3.NewDatabase(conMan, dbProperties) case dbProperties.ConnectionString.IsPostgres(): return nil, fmt.Errorf("can't use Postgres implementation") default: diff --git a/relayapi/relayapi.go b/relayapi/relayapi.go index 200a1814a1..925fc031da 100644 --- a/relayapi/relayapi.go +++ b/relayapi/relayapi.go @@ -16,6 +16,7 @@ package relayapi import ( "github.com/matrix-org/dendrite/federationapi/producers" + "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/relayapi/api" "github.com/matrix-org/dendrite/relayapi/internal" "github.com/matrix-org/dendrite/relayapi/routing" @@ -41,7 +42,7 @@ func AddPublicRoutes( } routing.Setup( - base.PublicFederationAPIMux, + base.Routers.Federation, fedCfg, relay, keyRing, @@ -55,10 +56,10 @@ func NewRelayInternalAPI( keyRing *gomatrixserverlib.KeyRing, producer *producers.SyncAPIProducer, relayingEnabled bool, + caches caching.FederationCache, ) api.RelayInternalAPI { cfg := &base.Cfg.RelayAPI - - relayDB, err := storage.NewDatabase(base, &cfg.Database, base.Caches, base.Cfg.Global.IsLocalServerName) + relayDB, err := storage.NewDatabase(base.ConnectionManager, &cfg.Database, caches, base.Cfg.Global.IsLocalServerName) if err != nil { logrus.WithError(err).Panic("failed to connect to relay db") } diff --git a/relayapi/relayapi_test.go b/relayapi/relayapi_test.go index f1b3262aa7..e812030988 100644 --- a/relayapi/relayapi_test.go +++ b/relayapi/relayapi_test.go @@ -24,6 +24,7 @@ import ( "github.com/gorilla/mux" "github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing" + "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/relayapi" "github.com/matrix-org/dendrite/test" "github.com/matrix-org/dendrite/test/testrig" @@ -34,9 +35,10 @@ import ( func TestCreateNewRelayInternalAPI(t *testing.T) { test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { base, close := testrig.CreateBaseDendrite(t, dbType) + caches := caching.NewRistrettoCache(base.Cfg.Global.Cache.EstimatedMaxSize, base.Cfg.Global.Cache.MaxAge, caching.DisableMetrics) defer close() - relayAPI := relayapi.NewRelayInternalAPI(base, nil, nil, nil, nil, true) + relayAPI := relayapi.NewRelayInternalAPI(base, nil, nil, nil, nil, true, caches) assert.NotNil(t, relayAPI) }) } @@ -52,7 +54,7 @@ func TestCreateRelayInternalInvalidDatabasePanics(t *testing.T) { defer close() assert.Panics(t, func() { - relayapi.NewRelayInternalAPI(base, nil, nil, nil, nil, true) + relayapi.NewRelayInternalAPI(base, nil, nil, nil, nil, true, nil) }) }) } @@ -107,8 +109,9 @@ func TestCreateRelayPublicRoutes(t *testing.T) { test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { base, close := testrig.CreateBaseDendrite(t, dbType) defer close() + caches := caching.NewRistrettoCache(base.Cfg.Global.Cache.EstimatedMaxSize, base.Cfg.Global.Cache.MaxAge, caching.DisableMetrics) - relayAPI := relayapi.NewRelayInternalAPI(base, nil, nil, nil, nil, true) + relayAPI := relayapi.NewRelayInternalAPI(base, nil, nil, nil, nil, true, caches) assert.NotNil(t, relayAPI) serverKeyAPI := &signing.YggdrasilKeys{} @@ -144,7 +147,7 @@ func TestCreateRelayPublicRoutes(t *testing.T) { for _, tc := range testCases { w := httptest.NewRecorder() - base.PublicFederationAPIMux.ServeHTTP(w, tc.req) + base.Routers.Federation.ServeHTTP(w, tc.req) if w.Code != tc.wantCode { t.Fatalf("%s: got HTTP %d want %d", tc.name, w.Code, tc.wantCode) } @@ -156,8 +159,9 @@ func TestDisableRelayPublicRoutes(t *testing.T) { test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { base, close := testrig.CreateBaseDendrite(t, dbType) defer close() + caches := caching.NewRistrettoCache(base.Cfg.Global.Cache.EstimatedMaxSize, base.Cfg.Global.Cache.MaxAge, caching.DisableMetrics) - relayAPI := relayapi.NewRelayInternalAPI(base, nil, nil, nil, nil, false) + relayAPI := relayapi.NewRelayInternalAPI(base, nil, nil, nil, nil, false, caches) assert.NotNil(t, relayAPI) serverKeyAPI := &signing.YggdrasilKeys{} @@ -183,7 +187,7 @@ func TestDisableRelayPublicRoutes(t *testing.T) { for _, tc := range testCases { w := httptest.NewRecorder() - base.PublicFederationAPIMux.ServeHTTP(w, tc.req) + base.Routers.Federation.ServeHTTP(w, tc.req) if w.Code != tc.wantCode { t.Fatalf("%s: got HTTP %d want %d", tc.name, w.Code, tc.wantCode) } diff --git a/relayapi/storage/postgres/storage.go b/relayapi/storage/postgres/storage.go index 1042beba77..78bbaf1c26 100644 --- a/relayapi/storage/postgres/storage.go +++ b/relayapi/storage/postgres/storage.go @@ -20,7 +20,6 @@ import ( "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/relayapi/storage/shared" - "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/gomatrixserverlib" ) @@ -34,14 +33,14 @@ type Database struct { // NewDatabase opens a new database func NewDatabase( - base *base.BaseDendrite, + conMan sqlutil.Connections, dbProperties *config.DatabaseOptions, cache caching.FederationCache, isLocalServerName func(gomatrixserverlib.ServerName) bool, ) (*Database, error) { var d Database var err error - if d.db, d.writer, err = base.DatabaseConnection(dbProperties, sqlutil.NewDummyWriter()); err != nil { + if d.db, d.writer, err = conMan.Connection(dbProperties); err != nil { return nil, err } queue, err := NewPostgresRelayQueueTable(d.db) diff --git a/relayapi/storage/sqlite3/storage.go b/relayapi/storage/sqlite3/storage.go index 3ed4ab046e..da2cf9f7f1 100644 --- a/relayapi/storage/sqlite3/storage.go +++ b/relayapi/storage/sqlite3/storage.go @@ -20,7 +20,6 @@ import ( "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/relayapi/storage/shared" - "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/gomatrixserverlib" ) @@ -34,14 +33,14 @@ type Database struct { // NewDatabase opens a new database func NewDatabase( - base *base.BaseDendrite, + conMan sqlutil.Connections, dbProperties *config.DatabaseOptions, cache caching.FederationCache, isLocalServerName func(gomatrixserverlib.ServerName) bool, ) (*Database, error) { var d Database var err error - if d.db, d.writer, err = base.DatabaseConnection(dbProperties, sqlutil.NewExclusiveWriter()); err != nil { + if d.db, d.writer, err = conMan.Connection(dbProperties); err != nil { return nil, err } queue, err := NewSQLiteRelayQueueTable(d.db) diff --git a/relayapi/storage/storage.go b/relayapi/storage/storage.go index 16ecbcfb72..17d9e6c759 100644 --- a/relayapi/storage/storage.go +++ b/relayapi/storage/storage.go @@ -21,25 +21,25 @@ import ( "fmt" "github.com/matrix-org/dendrite/internal/caching" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/relayapi/storage/postgres" "github.com/matrix-org/dendrite/relayapi/storage/sqlite3" - "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/gomatrixserverlib" ) // NewDatabase opens a new database func NewDatabase( - base *base.BaseDendrite, + conMan sqlutil.Connections, dbProperties *config.DatabaseOptions, cache caching.FederationCache, isLocalServerName func(gomatrixserverlib.ServerName) bool, ) (Database, error) { switch { case dbProperties.ConnectionString.IsSQLite(): - return sqlite3.NewDatabase(base, dbProperties, cache, isLocalServerName) + return sqlite3.NewDatabase(conMan, dbProperties, cache, isLocalServerName) case dbProperties.ConnectionString.IsPostgres(): - return postgres.NewDatabase(base, dbProperties, cache, isLocalServerName) + return postgres.NewDatabase(conMan, dbProperties, cache, isLocalServerName) default: return nil, fmt.Errorf("unexpected database type") } diff --git a/relayapi/storage/storage_wasm.go b/relayapi/storage/storage_wasm.go new file mode 100644 index 0000000000..92847e1bad --- /dev/null +++ b/relayapi/storage/storage_wasm.go @@ -0,0 +1,42 @@ +// Copyright 2022 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "fmt" + + "github.com/matrix-org/dendrite/internal/caching" + "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/relayapi/storage/sqlite3" + "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/gomatrixserverlib" +) + +// NewDatabase opens a new database +func NewDatabase( + conMan sqlutil.Connections, + dbProperties *config.DatabaseOptions, + cache caching.FederationCache, + isLocalServerName func(gomatrixserverlib.ServerName) bool, +) (Database, error) { + switch { + case dbProperties.ConnectionString.IsSQLite(): + return sqlite3.NewDatabase(conMan, dbProperties, cache, isLocalServerName) + case dbProperties.ConnectionString.IsPostgres(): + return nil, fmt.Errorf("can't use Postgres implementation") + default: + return nil, fmt.Errorf("unexpected database type") + } +} diff --git a/roomserver/internal/api.go b/roomserver/internal/api.go index c43b9d0498..2e987d681e 100644 --- a/roomserver/internal/api.go +++ b/roomserver/internal/api.go @@ -60,7 +60,7 @@ type RoomserverInternalAPI struct { func NewRoomserverAPI( base *base.BaseDendrite, roomserverDB storage.Database, - js nats.JetStreamContext, nc *nats.Conn, + js nats.JetStreamContext, nc *nats.Conn, caches caching.RoomServerCaches, ) *RoomserverInternalAPI { var perspectiveServerNames []gomatrixserverlib.ServerName for _, kp := range base.Cfg.FederationAPI.KeyPerspectives { @@ -78,7 +78,7 @@ func NewRoomserverAPI( DB: roomserverDB, Base: base, Cfg: &base.Cfg.RoomServer, - Cache: base.Caches, + Cache: caches, ServerName: base.Cfg.Global.ServerName, PerspectiveServerNames: perspectiveServerNames, InputRoomEventTopic: base.Cfg.Global.JetStream.Prefixed(jetstream.InputRoomEvent), @@ -89,7 +89,7 @@ func NewRoomserverAPI( ServerACLs: serverACLs, Queryer: &query.Queryer{ DB: roomserverDB, - Cache: base.Caches, + Cache: caches, IsLocalServerName: base.Cfg.Global.IsLocalServerName, ServerACLs: serverACLs, }, diff --git a/roomserver/internal/helpers/helpers_test.go b/roomserver/internal/helpers/helpers_test.go index c056e704c5..03a8bf5752 100644 --- a/roomserver/internal/helpers/helpers_test.go +++ b/roomserver/internal/helpers/helpers_test.go @@ -3,26 +3,28 @@ package helpers import ( "context" "testing" + "time" + "github.com/matrix-org/dendrite/internal/caching" + "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/setup/config" "github.com/stretchr/testify/assert" "github.com/matrix-org/dendrite/roomserver/types" - "github.com/matrix-org/dendrite/setup/base" - - "github.com/matrix-org/dendrite/test" - "github.com/matrix-org/dendrite/test/testrig" - "github.com/matrix-org/dendrite/roomserver/storage" + "github.com/matrix-org/dendrite/test" ) -func mustCreateDatabase(t *testing.T, dbType test.DBType) (*base.BaseDendrite, storage.Database, func()) { - base, close := testrig.CreateBaseDendrite(t, dbType) - db, err := storage.Open(base, &base.Cfg.RoomServer.Database, base.Caches) +func mustCreateDatabase(t *testing.T, dbType test.DBType) (storage.Database, func()) { + conStr, close := test.PrepareDBConnectionString(t, dbType) + caches := caching.NewRistrettoCache(8*1024*1024, time.Hour, caching.DisableMetrics) + cm := sqlutil.NewConnectionManager() + db, err := storage.Open(context.Background(), cm, &config.DatabaseOptions{ConnectionString: config.DataSource(conStr)}, caches) if err != nil { t.Fatalf("failed to create Database: %v", err) } - return base, db, close + return db, close } func TestIsInvitePendingWithoutNID(t *testing.T) { @@ -32,7 +34,7 @@ func TestIsInvitePendingWithoutNID(t *testing.T) { room := test.NewRoom(t, alice, test.RoomPreset(test.PresetPublicChat)) _ = bob test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { - _, db, close := mustCreateDatabase(t, dbType) + db, close := mustCreateDatabase(t, dbType) defer close() // store all events diff --git a/roomserver/internal/input/input_test.go b/roomserver/internal/input/input_test.go index 4708560ac2..555ec9c6dc 100644 --- a/roomserver/internal/input/input_test.go +++ b/roomserver/internal/input/input_test.go @@ -7,6 +7,7 @@ import ( "time" "github.com/matrix-org/dendrite/internal/caching" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/roomserver/internal/input" "github.com/matrix-org/dendrite/roomserver/storage" @@ -48,14 +49,15 @@ func TestSingleTransactionOnInput(t *testing.T) { Kind: api.KindOutlier, // don't panic if we generate an output event Event: event.Headered(gomatrixserverlib.RoomVersionV6), } + cm := sqlutil.NewConnectionManager() db, err := storage.Open( - nil, + context.Background(), cm, &config.DatabaseOptions{ ConnectionString: "", MaxOpenConnections: 1, MaxIdleConnections: 1, }, - caching.NewRistrettoCache(8*1024*1024, time.Hour, false), + caching.NewRistrettoCache(8*1024*1024, time.Hour, caching.DisableMetrics), ) if err != nil { t.Logf("PostgreSQL not available (%s), skipping", err) diff --git a/roomserver/roomserver.go b/roomserver/roomserver.go index 5a8d8b5702..1c55423ef1 100644 --- a/roomserver/roomserver.go +++ b/roomserver/roomserver.go @@ -15,6 +15,7 @@ package roomserver import ( + "github.com/matrix-org/dendrite/internal/caching" "github.com/sirupsen/logrus" "github.com/matrix-org/dendrite/roomserver/api" @@ -26,10 +27,11 @@ import ( // NewInternalAPI returns a concrete implementation of the internal API. func NewInternalAPI( base *base.BaseDendrite, + caches caching.RoomServerCaches, ) api.RoomserverInternalAPI { cfg := &base.Cfg.RoomServer - roomserverDB, err := storage.Open(base, &cfg.Database, base.Caches) + roomserverDB, err := storage.Open(base.ProcessContext.Context(), base.ConnectionManager, &cfg.Database, caches) if err != nil { logrus.WithError(err).Panicf("failed to connect to room server db") } @@ -37,6 +39,6 @@ func NewInternalAPI( js, nc := base.NATS.Prepare(base.ProcessContext, &cfg.Matrix.JetStream) return internal.NewRoomserverAPI( - base, roomserverDB, js, nc, + base, roomserverDB, js, nc, caches, ) } diff --git a/roomserver/roomserver_test.go b/roomserver/roomserver_test.go index a3ca5909ee..1b0b3155db 100644 --- a/roomserver/roomserver_test.go +++ b/roomserver/roomserver_test.go @@ -7,6 +7,7 @@ import ( "testing" "time" + "github.com/matrix-org/dendrite/internal/caching" "github.com/stretchr/testify/assert" "github.com/matrix-org/dendrite/roomserver/state" @@ -32,7 +33,8 @@ import ( func mustCreateDatabase(t *testing.T, dbType test.DBType) (*base.BaseDendrite, storage.Database, func()) { t.Helper() base, close := testrig.CreateBaseDendrite(t, dbType) - db, err := storage.Open(base, &base.Cfg.RoomServer.Database, base.Caches) + caches := caching.NewRistrettoCache(base.Cfg.Global.Cache.EstimatedMaxSize, base.Cfg.Global.Cache.MaxAge, caching.DisableMetrics) + db, err := storage.Open(base.ProcessContext.Context(), base.ConnectionManager, &base.Cfg.RoomServer.Database, caches) if err != nil { t.Fatalf("failed to create Database: %v", err) } @@ -43,7 +45,8 @@ func TestUsers(t *testing.T) { test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { base, close := testrig.CreateBaseDendrite(t, dbType) defer close() - rsAPI := roomserver.NewInternalAPI(base) + caches := caching.NewRistrettoCache(base.Cfg.Global.Cache.EstimatedMaxSize, base.Cfg.Global.Cache.MaxAge, caching.DisableMetrics) + rsAPI := roomserver.NewInternalAPI(base, caches) // SetFederationAPI starts the room event input consumer rsAPI.SetFederationAPI(nil, nil) @@ -181,7 +184,8 @@ func Test_QueryLeftUsers(t *testing.T) { base, _, close := mustCreateDatabase(t, dbType) defer close() - rsAPI := roomserver.NewInternalAPI(base) + caches := caching.NewRistrettoCache(base.Cfg.Global.Cache.EstimatedMaxSize, base.Cfg.Global.Cache.MaxAge, caching.DisableMetrics) + rsAPI := roomserver.NewInternalAPI(base, caches) // SetFederationAPI starts the room event input consumer rsAPI.SetFederationAPI(nil, nil) // Create the room @@ -232,12 +236,13 @@ func TestPurgeRoom(t *testing.T) { defer jetstream.DeleteAllStreams(jsCtx, &base.Cfg.Global.JetStream) fedClient := base.CreateFederationClient() - rsAPI := roomserver.NewInternalAPI(base) + caches := caching.NewRistrettoCache(base.Cfg.Global.Cache.EstimatedMaxSize, base.Cfg.Global.Cache.MaxAge, caching.DisableMetrics) + rsAPI := roomserver.NewInternalAPI(base, caches) userAPI := userapi.NewInternalAPI(base, rsAPI, nil) // this starts the JetStream consumers - syncapi.AddPublicRoutes(base, userAPI, rsAPI) - federationapi.NewInternalAPI(base, fedClient, rsAPI, base.Caches, nil, true) + syncapi.AddPublicRoutes(base, userAPI, rsAPI, caches) + federationapi.NewInternalAPI(base, fedClient, rsAPI, caches, nil, true) rsAPI.SetFederationAPI(nil, nil) // Create the room diff --git a/roomserver/storage/postgres/storage.go b/roomserver/storage/postgres/storage.go index d98a5cf979..19cde54105 100644 --- a/roomserver/storage/postgres/storage.go +++ b/roomserver/storage/postgres/storage.go @@ -28,7 +28,6 @@ import ( "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver/storage/postgres/deltas" "github.com/matrix-org/dendrite/roomserver/storage/shared" - "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/config" ) @@ -38,10 +37,10 @@ type Database struct { } // Open a postgres database. -func Open(base *base.BaseDendrite, dbProperties *config.DatabaseOptions, cache caching.RoomServerCaches) (*Database, error) { +func Open(ctx context.Context, conMan sqlutil.Connections, dbProperties *config.DatabaseOptions, cache caching.RoomServerCaches) (*Database, error) { var d Database var err error - db, writer, err := base.DatabaseConnection(dbProperties, sqlutil.NewDummyWriter()) + db, writer, err := conMan.Connection(dbProperties) if err != nil { return nil, fmt.Errorf("sqlutil.Open: %w", err) } @@ -53,7 +52,7 @@ func Open(base *base.BaseDendrite, dbProperties *config.DatabaseOptions, cache c // Special case, since this migration uses several tables, so it needs to // be sure that all tables are created first. - if err = executeMigration(base.Context(), db); err != nil { + if err = executeMigration(ctx, db); err != nil { return nil, err } diff --git a/roomserver/storage/sqlite3/storage.go b/roomserver/storage/sqlite3/storage.go index 2adedd2d8e..89e16fc141 100644 --- a/roomserver/storage/sqlite3/storage.go +++ b/roomserver/storage/sqlite3/storage.go @@ -28,7 +28,6 @@ import ( "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/sqlite3/deltas" "github.com/matrix-org/dendrite/roomserver/types" - "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/config" ) @@ -38,10 +37,10 @@ type Database struct { } // Open a sqlite database. -func Open(base *base.BaseDendrite, dbProperties *config.DatabaseOptions, cache caching.RoomServerCaches) (*Database, error) { +func Open(ctx context.Context, conMan sqlutil.Connections, dbProperties *config.DatabaseOptions, cache caching.RoomServerCaches) (*Database, error) { var d Database var err error - db, writer, err := base.DatabaseConnection(dbProperties, sqlutil.NewExclusiveWriter()) + db, writer, err := conMan.Connection(dbProperties) if err != nil { return nil, fmt.Errorf("sqlutil.Open: %w", err) } @@ -62,7 +61,7 @@ func Open(base *base.BaseDendrite, dbProperties *config.DatabaseOptions, cache c // Special case, since this migration uses several tables, so it needs to // be sure that all tables are created first. - if err = executeMigration(base.Context(), db); err != nil { + if err = executeMigration(ctx, db); err != nil { return nil, err } diff --git a/roomserver/storage/storage.go b/roomserver/storage/storage.go index 8a87b7d7c6..2b3b3bd85b 100644 --- a/roomserver/storage/storage.go +++ b/roomserver/storage/storage.go @@ -18,22 +18,23 @@ package storage import ( + "context" "fmt" "github.com/matrix-org/dendrite/internal/caching" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver/storage/postgres" "github.com/matrix-org/dendrite/roomserver/storage/sqlite3" - "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/config" ) // Open opens a database connection. -func Open(base *base.BaseDendrite, dbProperties *config.DatabaseOptions, cache caching.RoomServerCaches) (Database, error) { +func Open(ctx context.Context, conMan sqlutil.Connections, dbProperties *config.DatabaseOptions, cache caching.RoomServerCaches) (Database, error) { switch { case dbProperties.ConnectionString.IsSQLite(): - return sqlite3.Open(base, dbProperties, cache) + return sqlite3.Open(ctx, conMan, dbProperties, cache) case dbProperties.ConnectionString.IsPostgres(): - return postgres.Open(base, dbProperties, cache) + return postgres.Open(ctx, conMan, dbProperties, cache) default: return nil, fmt.Errorf("unexpected database type") } diff --git a/roomserver/storage/storage_wasm.go b/roomserver/storage/storage_wasm.go index df5a56ac37..817f9304c7 100644 --- a/roomserver/storage/storage_wasm.go +++ b/roomserver/storage/storage_wasm.go @@ -15,19 +15,20 @@ package storage import ( + "context" "fmt" "github.com/matrix-org/dendrite/internal/caching" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver/storage/sqlite3" - "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/config" ) // NewPublicRoomsServerDatabase opens a database connection. -func Open(base *base.BaseDendrite, dbProperties *config.DatabaseOptions, cache caching.RoomServerCaches) (Database, error) { +func Open(ctx context.Context, conMan sqlutil.Connections, dbProperties *config.DatabaseOptions, cache caching.RoomServerCaches) (Database, error) { switch { case dbProperties.ConnectionString.IsSQLite(): - return sqlite3.Open(base, dbProperties, cache) + return sqlite3.Open(ctx, conMan, dbProperties, cache) case dbProperties.ConnectionString.IsPostgres(): return nil, fmt.Errorf("can't use Postgres implementation") default: diff --git a/setup/base/base.go b/setup/base/base.go index dfe48ff3cb..8c9b06d0e9 100644 --- a/setup/base/base.go +++ b/setup/base/base.go @@ -17,7 +17,6 @@ package base import ( "bytes" "context" - "database/sql" "embed" "encoding/json" "errors" @@ -36,15 +35,13 @@ import ( "github.com/getsentry/sentry-go" sentryhttp "github.com/getsentry/sentry-go/http" + "github.com/matrix-org/dendrite/setup/jetstream" "github.com/matrix-org/gomatrixserverlib" "github.com/prometheus/client_golang/prometheus/promhttp" "go.uber.org/atomic" "github.com/matrix-org/dendrite/internal" - "github.com/matrix-org/dendrite/internal/caching" - "github.com/matrix-org/dendrite/internal/fulltext" "github.com/matrix-org/dendrite/internal/httputil" - "github.com/matrix-org/dendrite/internal/pushgateway" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/gorilla/mux" @@ -53,7 +50,6 @@ import ( "github.com/sirupsen/logrus" "github.com/matrix-org/dendrite/setup/config" - "github.com/matrix-org/dendrite/setup/jetstream" "github.com/matrix-org/dendrite/setup/process" ) @@ -67,24 +63,14 @@ var staticContent embed.FS // Must be closed when shutting down. type BaseDendrite struct { *process.ProcessContext - tracerCloser io.Closer - PublicClientAPIMux *mux.Router - PublicFederationAPIMux *mux.Router - PublicKeyAPIMux *mux.Router - PublicMediaAPIMux *mux.Router - PublicWellKnownAPIMux *mux.Router - PublicStaticMux *mux.Router - DendriteAdminMux *mux.Router - SynapseAdminMux *mux.Router - NATS *jetstream.NATSInstance - Cfg *config.Dendrite - Caches *caching.Caches - DNSCache *gomatrixserverlib.DNSCache - Database *sql.DB - DatabaseWriter sqlutil.Writer - EnableMetrics bool - Fulltext *fulltext.Search - startupLock sync.Mutex + tracerCloser io.Closer + Routers httputil.Routers + NATS *jetstream.NATSInstance + Cfg *config.Dendrite + DNSCache *gomatrixserverlib.DNSCache + ConnectionManager sqlutil.Connections + EnableMetrics bool + startupLock sync.Mutex } const HTTPServerTimeout = time.Minute * 5 @@ -130,14 +116,6 @@ func NewBaseDendrite(cfg *config.Dendrite, options ...BaseDendriteOptions) *Base logrus.WithError(err).Panicf("failed to start opentracing") } - var fts *fulltext.Search - if cfg.SyncAPI.Fulltext.Enabled { - fts, err = fulltext.New(cfg.SyncAPI.Fulltext) - if err != nil { - logrus.WithError(err).Panicf("failed to create full text") - } - } - if cfg.Global.Sentry.Enabled { logrus.Info("Setting up Sentry for debugging...") err = sentry.Init(sentry.ClientOptions{ @@ -169,14 +147,13 @@ func NewBaseDendrite(cfg *config.Dendrite, options ...BaseDendriteOptions) *Base // If we're in monolith mode, we'll set up a global pool of database // connections. A component is welcome to use this pool if they don't // have a separate database config of their own. - var db *sql.DB - var writer sqlutil.Writer + cm := sqlutil.NewConnectionManager() if cfg.Global.DatabaseOptions.ConnectionString != "" { if cfg.Global.DatabaseOptions.ConnectionString.IsSQLite() { logrus.Panic("Using a global database connection pool is not supported with SQLite databases") } - writer = sqlutil.NewDummyWriter() - if db, err = sqlutil.Open(&cfg.Global.DatabaseOptions, writer); err != nil { + _, _, err := cm.Connection(&cfg.Global.DatabaseOptions) + if err != nil { logrus.WithError(err).Panic("Failed to set up global database connections") } logrus.Debug("Using global database connection pool") @@ -195,24 +172,14 @@ func NewBaseDendrite(cfg *config.Dendrite, options ...BaseDendriteOptions) *Base // directory traversal attack e.g /../../../etc/passwd return &BaseDendrite{ - ProcessContext: process.NewProcessContext(), - tracerCloser: closer, - Cfg: cfg, - Caches: caching.NewRistrettoCache(cfg.Global.Cache.EstimatedMaxSize, cfg.Global.Cache.MaxAge, enableMetrics), - DNSCache: dnsCache, - PublicClientAPIMux: mux.NewRouter().SkipClean(true).PathPrefix(httputil.PublicClientPathPrefix).Subrouter().UseEncodedPath(), - PublicFederationAPIMux: mux.NewRouter().SkipClean(true).PathPrefix(httputil.PublicFederationPathPrefix).Subrouter().UseEncodedPath(), - PublicKeyAPIMux: mux.NewRouter().SkipClean(true).PathPrefix(httputil.PublicKeyPathPrefix).Subrouter().UseEncodedPath(), - PublicMediaAPIMux: mux.NewRouter().SkipClean(true).PathPrefix(httputil.PublicMediaPathPrefix).Subrouter().UseEncodedPath(), - PublicWellKnownAPIMux: mux.NewRouter().SkipClean(true).PathPrefix(httputil.PublicWellKnownPrefix).Subrouter().UseEncodedPath(), - PublicStaticMux: mux.NewRouter().SkipClean(true).PathPrefix(httputil.PublicStaticPath).Subrouter().UseEncodedPath(), - DendriteAdminMux: mux.NewRouter().SkipClean(true).PathPrefix(httputil.DendriteAdminPathPrefix).Subrouter().UseEncodedPath(), - SynapseAdminMux: mux.NewRouter().SkipClean(true).PathPrefix(httputil.SynapseAdminPathPrefix).Subrouter().UseEncodedPath(), - NATS: &jetstream.NATSInstance{}, - Database: db, // set if monolith with global connection pool only - DatabaseWriter: writer, // set if monolith with global connection pool only - EnableMetrics: enableMetrics, - Fulltext: fts, + ProcessContext: process.NewProcessContext(), + tracerCloser: closer, + Cfg: cfg, + DNSCache: dnsCache, + Routers: httputil.NewRouters(), + NATS: &jetstream.NATSInstance{}, + ConnectionManager: cm, + EnableMetrics: enableMetrics, } } @@ -223,34 +190,6 @@ func (b *BaseDendrite) Close() error { return b.tracerCloser.Close() } -// DatabaseConnection assists in setting up a database connection. It accepts -// the database properties and a new writer for the given component. If we're -// running in monolith mode with a global connection pool configured then we -// will return that connection, along with the global writer, effectively -// ignoring the options provided. Otherwise we'll open a new database connection -// using the supplied options and writer. Note that it's possible for the pointer -// receiver to be nil here – that's deliberate as some of the unit tests don't -// have a BaseDendrite and just want a connection with the supplied config -// without any pooling stuff. -func (b *BaseDendrite) DatabaseConnection(dbProperties *config.DatabaseOptions, writer sqlutil.Writer) (*sql.DB, sqlutil.Writer, error) { - if dbProperties.ConnectionString != "" || b == nil { - // Open a new database connection using the supplied config. - db, err := sqlutil.Open(dbProperties, writer) - return db, writer, err - } - if b.Database != nil && b.DatabaseWriter != nil { - // Ignore the supplied config and return the global pool and - // writer. - return b.Database, b.DatabaseWriter, nil - } - return nil, nil, fmt.Errorf("no database connections configured") -} - -// PushGatewayHTTPClient returns a new client for interacting with (external) Push Gateways. -func (b *BaseDendrite) PushGatewayHTTPClient() pushgateway.Client { - return pushgateway.NewHTTPClient(b.Cfg.UserAPI.PushGatewayDisableTLSValidation) -} - // CreateClient creates a new client (normally used for media fetch requests). // Should only be called once per component. func (b *BaseDendrite) CreateClient() *gomatrixserverlib.Client { @@ -295,40 +234,11 @@ func (b *BaseDendrite) CreateFederationClient() *gomatrixserverlib.FederationCli return client } -func (b *BaseDendrite) configureHTTPErrors() { - notAllowedHandler := func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusMethodNotAllowed) - _, _ = w.Write([]byte(fmt.Sprintf("405 %s not allowed on this endpoint", r.Method))) - } - - clientNotFoundHandler := func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusNotFound) - w.Header().Set("Content-Type", "application/json") - _, _ = w.Write([]byte(`{"errcode":"M_UNRECOGNIZED","error":"Unrecognized request"}`)) // nolint:misspell - } - - notFoundCORSHandler := httputil.WrapHandlerInCORS(http.NotFoundHandler()) - notAllowedCORSHandler := httputil.WrapHandlerInCORS(http.HandlerFunc(notAllowedHandler)) - - for _, router := range []*mux.Router{ - b.PublicMediaAPIMux, b.DendriteAdminMux, - b.SynapseAdminMux, b.PublicWellKnownAPIMux, - b.PublicStaticMux, - } { - router.NotFoundHandler = notFoundCORSHandler - router.MethodNotAllowedHandler = notAllowedCORSHandler - } - - // Special case so that we don't upset clients on the CS API. - b.PublicClientAPIMux.NotFoundHandler = http.HandlerFunc(clientNotFoundHandler) - b.PublicClientAPIMux.MethodNotAllowedHandler = http.HandlerFunc(clientNotFoundHandler) -} - func (b *BaseDendrite) ConfigureAdminEndpoints() { - b.DendriteAdminMux.HandleFunc("/monitor/up", func(w http.ResponseWriter, r *http.Request) { + b.Routers.DendriteAdmin.HandleFunc("/monitor/up", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(200) }) - b.DendriteAdminMux.HandleFunc("/monitor/health", func(w http.ResponseWriter, r *http.Request) { + b.Routers.DendriteAdmin.HandleFunc("/monitor/health", func(w http.ResponseWriter, r *http.Request) { if isDegraded, reasons := b.ProcessContext.IsDegraded(); isDegraded { w.WriteHeader(503) _ = json.NewEncoder(w).Encode(struct { @@ -363,8 +273,6 @@ func (b *BaseDendrite) SetupAndServeHTTP( }, } - b.configureHTTPErrors() - //Redirect for Landing Page externalRouter.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { http.Redirect(w, r, httputil.PublicStaticPath, http.StatusFound) @@ -385,39 +293,42 @@ func (b *BaseDendrite) SetupAndServeHTTP( logrus.WithError(err).Fatal("failed to execute landing page template") } - b.PublicStaticMux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + b.Routers.Static.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write(landingPage.Bytes()) }) var clientHandler http.Handler - clientHandler = b.PublicClientAPIMux + clientHandler = b.Routers.Client if b.Cfg.Global.Sentry.Enabled { sentryHandler := sentryhttp.New(sentryhttp.Options{ Repanic: true, }) - clientHandler = sentryHandler.Handle(b.PublicClientAPIMux) + clientHandler = sentryHandler.Handle(b.Routers.Client) } var federationHandler http.Handler - federationHandler = b.PublicFederationAPIMux + federationHandler = b.Routers.Federation if b.Cfg.Global.Sentry.Enabled { sentryHandler := sentryhttp.New(sentryhttp.Options{ Repanic: true, }) - federationHandler = sentryHandler.Handle(b.PublicFederationAPIMux) + federationHandler = sentryHandler.Handle(b.Routers.Federation) } - externalRouter.PathPrefix(httputil.DendriteAdminPathPrefix).Handler(b.DendriteAdminMux) + externalRouter.PathPrefix(httputil.DendriteAdminPathPrefix).Handler(b.Routers.DendriteAdmin) externalRouter.PathPrefix(httputil.PublicClientPathPrefix).Handler(clientHandler) if !b.Cfg.Global.DisableFederation { - externalRouter.PathPrefix(httputil.PublicKeyPathPrefix).Handler(b.PublicKeyAPIMux) + externalRouter.PathPrefix(httputil.PublicKeyPathPrefix).Handler(b.Routers.Keys) externalRouter.PathPrefix(httputil.PublicFederationPathPrefix).Handler(federationHandler) } - externalRouter.PathPrefix(httputil.SynapseAdminPathPrefix).Handler(b.SynapseAdminMux) - externalRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(b.PublicMediaAPIMux) - externalRouter.PathPrefix(httputil.PublicWellKnownPrefix).Handler(b.PublicWellKnownAPIMux) - externalRouter.PathPrefix(httputil.PublicStaticPath).Handler(b.PublicStaticMux) + externalRouter.PathPrefix(httputil.SynapseAdminPathPrefix).Handler(b.Routers.SynapseAdmin) + externalRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(b.Routers.Media) + externalRouter.PathPrefix(httputil.PublicWellKnownPrefix).Handler(b.Routers.WellKnown) + externalRouter.PathPrefix(httputil.PublicStaticPath).Handler(b.Routers.Static) b.startupLock.Unlock() + externalRouter.NotFoundHandler = httputil.NotFoundCORSHandler + externalRouter.MethodNotAllowedHandler = httputil.NotAllowedHandler + if externalHTTPAddr.Enabled() { go func() { var externalShutdown atomic.Bool // RegisterOnShutdown can be called more than once @@ -493,12 +404,6 @@ func (b *BaseDendrite) WaitForShutdown() { logrus.Warnf("failed to flush all Sentry events!") } } - if b.Fulltext != nil { - err := b.Fulltext.Close() - if err != nil { - logrus.Warnf("failed to close full text search!") - } - } logrus.Warnf("Dendrite is exiting now") } diff --git a/setup/jetstream/nats.go b/setup/jetstream/nats.go index 01fec9ad62..48683789bd 100644 --- a/setup/jetstream/nats.go +++ b/setup/jetstream/nats.go @@ -20,6 +20,8 @@ import ( type NATSInstance struct { *natsserver.Server + nc *natsclient.Conn + js natsclient.JetStreamContext } var natsLock sync.Mutex @@ -69,11 +71,18 @@ func (s *NATSInstance) Prepare(process *process.ProcessContext, cfg *config.JetS if !s.ReadyForConnections(time.Second * 10) { logrus.Fatalln("NATS did not start in time") } + // reuse existing connections + if s.nc != nil { + return s.js, s.nc + } nc, err := natsclient.Connect("", natsclient.InProcessServer(s)) if err != nil { logrus.Fatalln("Failed to create NATS client") } - return setupNATS(process, cfg, nc) + js, _ := setupNATS(process, cfg, nc) + s.js = js + s.nc = nc + return js, nc } func setupNATS(process *process.ProcessContext, cfg *config.JetStream, nc *natsclient.Conn) (natsclient.JetStreamContext, *natsclient.Conn) { diff --git a/setup/monolith.go b/setup/monolith.go index d8c652234f..174eba680f 100644 --- a/setup/monolith.go +++ b/setup/monolith.go @@ -20,6 +20,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/api" "github.com/matrix-org/dendrite/federationapi" federationAPI "github.com/matrix-org/dendrite/federationapi/api" + "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/internal/transactions" "github.com/matrix-org/dendrite/mediaapi" "github.com/matrix-org/dendrite/relayapi" @@ -52,7 +53,7 @@ type Monolith struct { } // AddAllPublicRoutes attaches all public paths to the given router -func (m *Monolith) AddAllPublicRoutes(base *base.BaseDendrite) { +func (m *Monolith) AddAllPublicRoutes(base *base.BaseDendrite, caches *caching.Caches) { userDirectoryProvider := m.ExtUserDirectoryProvider if userDirectoryProvider == nil { userDirectoryProvider = m.UserAPI @@ -66,7 +67,7 @@ func (m *Monolith) AddAllPublicRoutes(base *base.BaseDendrite) { base, m.UserAPI, m.FedClient, m.KeyRing, m.RoomserverAPI, m.FederationAPI, nil, ) mediaapi.AddPublicRoutes(base, m.UserAPI, m.Client) - syncapi.AddPublicRoutes(base, m.UserAPI, m.RoomserverAPI) + syncapi.AddPublicRoutes(base, m.UserAPI, m.RoomserverAPI, caches) if m.RelayAPI != nil { relayapi.AddPublicRoutes(base, m.KeyRing, m.RelayAPI) diff --git a/setup/mscs/msc2836/msc2836.go b/setup/mscs/msc2836/msc2836.go index 4bb6a5eeee..7c1e0fc61f 100644 --- a/setup/mscs/msc2836/msc2836.go +++ b/setup/mscs/msc2836/msc2836.go @@ -102,7 +102,7 @@ func Enable( base *base.BaseDendrite, rsAPI roomserver.RoomserverInternalAPI, fsAPI fs.FederationInternalAPI, userAPI userapi.UserInternalAPI, keyRing gomatrixserverlib.JSONVerifier, ) error { - db, err := NewDatabase(base, &base.Cfg.MSCs.Database) + db, err := NewDatabase(base.ConnectionManager, &base.Cfg.MSCs.Database) if err != nil { return fmt.Errorf("cannot enable MSC2836: %w", err) } @@ -125,11 +125,11 @@ func Enable( } }) - base.PublicClientAPIMux.Handle("/unstable/event_relationships", + base.Routers.Client.Handle("/unstable/event_relationships", httputil.MakeAuthAPI("eventRelationships", userAPI, eventRelationshipHandler(db, rsAPI, fsAPI)), ).Methods(http.MethodPost, http.MethodOptions) - base.PublicFederationAPIMux.Handle("/unstable/event_relationships", httputil.MakeExternalAPI( + base.Routers.Federation.Handle("/unstable/event_relationships", httputil.MakeExternalAPI( "msc2836_event_relationships", func(req *http.Request) util.JSONResponse { fedReq, errResp := gomatrixserverlib.VerifyHTTPRequest( req, time.Now(), base.Cfg.Global.ServerName, base.Cfg.Global.IsLocalServerName, keyRing, diff --git a/setup/mscs/msc2836/msc2836_test.go b/setup/mscs/msc2836/msc2836_test.go index f12fbbfcb6..24e96f931f 100644 --- a/setup/mscs/msc2836/msc2836_test.go +++ b/setup/mscs/msc2836/msc2836_test.go @@ -19,6 +19,7 @@ import ( "github.com/matrix-org/dendrite/internal/hooks" "github.com/matrix-org/dendrite/internal/httputil" + "github.com/matrix-org/dendrite/internal/sqlutil" roomserver "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/config" @@ -554,10 +555,11 @@ func injectEvents(t *testing.T, userAPI userapi.UserInternalAPI, rsAPI roomserve cfg.Global.ServerName = "localhost" cfg.MSCs.Database.ConnectionString = "file:msc2836_test.db" cfg.MSCs.MSCs = []string{"msc2836"} + cm := sqlutil.NewConnectionManager() base := &base.BaseDendrite{ - Cfg: cfg, - PublicClientAPIMux: mux.NewRouter().PathPrefix(httputil.PublicClientPathPrefix).Subrouter(), - PublicFederationAPIMux: mux.NewRouter().PathPrefix(httputil.PublicFederationPathPrefix).Subrouter(), + Cfg: cfg, + Routers: httputil.NewRouters(), + ConnectionManager: cm, } err := msc2836.Enable(base, rsAPI, nil, userAPI, nil) @@ -567,7 +569,7 @@ func injectEvents(t *testing.T, userAPI userapi.UserInternalAPI, rsAPI roomserve for _, ev := range events { hooks.Run(hooks.KindNewEventPersisted, ev) } - return base.PublicClientAPIMux + return base.Routers.Client } type fledglingEvent struct { diff --git a/setup/mscs/msc2836/storage.go b/setup/mscs/msc2836/storage.go index 827e82f703..1cf7e87856 100644 --- a/setup/mscs/msc2836/storage.go +++ b/setup/mscs/msc2836/storage.go @@ -8,7 +8,6 @@ import ( "encoding/json" "github.com/matrix-org/dendrite/internal/sqlutil" - "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" @@ -59,17 +58,17 @@ type DB struct { } // NewDatabase loads the database for msc2836 -func NewDatabase(base *base.BaseDendrite, dbOpts *config.DatabaseOptions) (Database, error) { +func NewDatabase(conMan sqlutil.Connections, dbOpts *config.DatabaseOptions) (Database, error) { if dbOpts.ConnectionString.IsPostgres() { - return newPostgresDatabase(base, dbOpts) + return newPostgresDatabase(conMan, dbOpts) } - return newSQLiteDatabase(base, dbOpts) + return newSQLiteDatabase(conMan, dbOpts) } -func newPostgresDatabase(base *base.BaseDendrite, dbOpts *config.DatabaseOptions) (Database, error) { +func newPostgresDatabase(conMan sqlutil.Connections, dbOpts *config.DatabaseOptions) (Database, error) { d := DB{} var err error - if d.db, d.writer, err = base.DatabaseConnection(dbOpts, sqlutil.NewDummyWriter()); err != nil { + if d.db, d.writer, err = conMan.Connection(dbOpts); err != nil { return nil, err } _, err = d.db.Exec(` @@ -144,10 +143,10 @@ func newPostgresDatabase(base *base.BaseDendrite, dbOpts *config.DatabaseOptions return &d, err } -func newSQLiteDatabase(base *base.BaseDendrite, dbOpts *config.DatabaseOptions) (Database, error) { +func newSQLiteDatabase(conMan sqlutil.Connections, dbOpts *config.DatabaseOptions) (Database, error) { d := DB{} var err error - if d.db, d.writer, err = base.DatabaseConnection(dbOpts, sqlutil.NewExclusiveWriter()); err != nil { + if d.db, d.writer, err = conMan.Connection(dbOpts); err != nil { return nil, err } _, err = d.db.Exec(` diff --git a/setup/mscs/msc2946/msc2946.go b/setup/mscs/msc2946/msc2946.go index 56c0635986..b4b93ff391 100644 --- a/setup/mscs/msc2946/msc2946.go +++ b/setup/mscs/msc2946/msc2946.go @@ -58,8 +58,8 @@ func Enable( fsAPI fs.FederationInternalAPI, keyRing gomatrixserverlib.JSONVerifier, cache caching.SpaceSummaryRoomsCache, ) error { clientAPI := httputil.MakeAuthAPI("spaces", userAPI, spacesHandler(rsAPI, fsAPI, cache, base.Cfg.Global.ServerName), httputil.WithAllowGuests()) - base.PublicClientAPIMux.Handle("/v1/rooms/{roomID}/hierarchy", clientAPI).Methods(http.MethodGet, http.MethodOptions) - base.PublicClientAPIMux.Handle("/unstable/org.matrix.msc2946/rooms/{roomID}/hierarchy", clientAPI).Methods(http.MethodGet, http.MethodOptions) + base.Routers.Client.Handle("/v1/rooms/{roomID}/hierarchy", clientAPI).Methods(http.MethodGet, http.MethodOptions) + base.Routers.Client.Handle("/unstable/org.matrix.msc2946/rooms/{roomID}/hierarchy", clientAPI).Methods(http.MethodGet, http.MethodOptions) fedAPI := httputil.MakeExternalAPI( "msc2946_fed_spaces", func(req *http.Request) util.JSONResponse { @@ -78,8 +78,8 @@ func Enable( return federatedSpacesHandler(req.Context(), fedReq, roomID, cache, rsAPI, fsAPI, base.Cfg.Global.ServerName) }, ) - base.PublicFederationAPIMux.Handle("/unstable/org.matrix.msc2946/hierarchy/{roomID}", fedAPI).Methods(http.MethodGet) - base.PublicFederationAPIMux.Handle("/v1/hierarchy/{roomID}", fedAPI).Methods(http.MethodGet) + base.Routers.Federation.Handle("/unstable/org.matrix.msc2946/hierarchy/{roomID}", fedAPI).Methods(http.MethodGet) + base.Routers.Federation.Handle("/v1/hierarchy/{roomID}", fedAPI).Methods(http.MethodGet) return nil } diff --git a/setup/mscs/mscs.go b/setup/mscs/mscs.go index 35b7bba3b9..b58c800b05 100644 --- a/setup/mscs/mscs.go +++ b/setup/mscs/mscs.go @@ -19,6 +19,7 @@ import ( "context" "fmt" + "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/setup" "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/mscs/msc2836" @@ -27,22 +28,22 @@ import ( ) // Enable MSCs - returns an error on unknown MSCs -func Enable(base *base.BaseDendrite, monolith *setup.Monolith) error { +func Enable(base *base.BaseDendrite, monolith *setup.Monolith, caches *caching.Caches) error { for _, msc := range base.Cfg.MSCs.MSCs { util.GetLogger(context.Background()).WithField("msc", msc).Info("Enabling MSC") - if err := EnableMSC(base, monolith, msc); err != nil { + if err := EnableMSC(base, monolith, msc, caches); err != nil { return err } } return nil } -func EnableMSC(base *base.BaseDendrite, monolith *setup.Monolith, msc string) error { +func EnableMSC(base *base.BaseDendrite, monolith *setup.Monolith, msc string, caches *caching.Caches) error { switch msc { case "msc2836": return msc2836.Enable(base, monolith.RoomserverAPI, monolith.FederationAPI, monolith.UserAPI, monolith.KeyRing) case "msc2946": - return msc2946.Enable(base, monolith.RoomserverAPI, monolith.UserAPI, monolith.FederationAPI, monolith.KeyRing, base.Caches) + return msc2946.Enable(base, monolith.RoomserverAPI, monolith.UserAPI, monolith.FederationAPI, monolith.KeyRing, caches) case "msc2444": // enabled inside federationapi case "msc2753": // enabled inside clientapi default: diff --git a/syncapi/consumers/clientapi.go b/syncapi/consumers/clientapi.go index 735f6718cc..43dc0f5176 100644 --- a/syncapi/consumers/clientapi.go +++ b/syncapi/consumers/clientapi.go @@ -50,7 +50,7 @@ type OutputClientDataConsumer struct { stream streams.StreamProvider notifier *notifier.Notifier serverName gomatrixserverlib.ServerName - fts *fulltext.Search + fts fulltext.Indexer cfg *config.SyncAPI } diff --git a/syncapi/consumers/roomserver.go b/syncapi/consumers/roomserver.go index a8d4d2b2c4..21f6104d61 100644 --- a/syncapi/consumers/roomserver.go +++ b/syncapi/consumers/roomserver.go @@ -51,7 +51,7 @@ type OutputRoomEventConsumer struct { pduStream streams.StreamProvider inviteStream streams.StreamProvider notifier *notifier.Notifier - fts *fulltext.Search + fts fulltext.Indexer } // NewOutputRoomEventConsumer creates a new OutputRoomEventConsumer. Call Start() to begin consuming from room servers. diff --git a/syncapi/routing/routing.go b/syncapi/routing/routing.go index 4cc1a6a85f..b1283247b5 100644 --- a/syncapi/routing/routing.go +++ b/syncapi/routing/routing.go @@ -43,7 +43,7 @@ func Setup( rsAPI api.SyncRoomserverAPI, cfg *config.SyncAPI, lazyLoadCache caching.LazyLoadCache, - fts *fulltext.Search, + fts fulltext.Indexer, ) { v1unstablemux := csMux.PathPrefix("/{apiversion:(?:v1|unstable)}/").Subrouter() v3mux := csMux.PathPrefix("/{apiversion:(?:r0|v3)}/").Subrouter() diff --git a/syncapi/routing/search.go b/syncapi/routing/search.go index 081ec6cb13..13625b9cbe 100644 --- a/syncapi/routing/search.go +++ b/syncapi/routing/search.go @@ -37,7 +37,7 @@ import ( ) // nolint:gocyclo -func Search(req *http.Request, device *api.Device, syncDB storage.Database, fts *fulltext.Search, from *string) util.JSONResponse { +func Search(req *http.Request, device *api.Device, syncDB storage.Database, fts fulltext.Indexer, from *string) util.JSONResponse { start := time.Now() var ( searchReq SearchRequest diff --git a/syncapi/storage/postgres/syncserver.go b/syncapi/storage/postgres/syncserver.go index 850d24a079..9f9de28d97 100644 --- a/syncapi/storage/postgres/syncserver.go +++ b/syncapi/storage/postgres/syncserver.go @@ -16,12 +16,12 @@ package postgres import ( + "context" "database/sql" // Import the postgres database driver. _ "github.com/lib/pq" "github.com/matrix-org/dendrite/internal/sqlutil" - "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/syncapi/storage/postgres/deltas" "github.com/matrix-org/dendrite/syncapi/storage/shared" @@ -36,10 +36,10 @@ type SyncServerDatasource struct { } // NewDatabase creates a new sync server database -func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions) (*SyncServerDatasource, error) { +func NewDatabase(ctx context.Context, cm sqlutil.Connections, dbProperties *config.DatabaseOptions) (*SyncServerDatasource, error) { var d SyncServerDatasource var err error - if d.db, d.writer, err = base.DatabaseConnection(dbProperties, sqlutil.NewDummyWriter()); err != nil { + if d.db, d.writer, err = cm.Connection(dbProperties); err != nil { return nil, err } accountData, err := NewPostgresAccountDataTable(d.db) @@ -111,7 +111,7 @@ func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions) Up: deltas.UpSetHistoryVisibility, // Requires current_room_state and output_room_events to be created. }, ) - err = m.Up(base.Context()) + err = m.Up(ctx) if err != nil { return nil, err } diff --git a/syncapi/storage/sqlite3/syncserver.go b/syncapi/storage/sqlite3/syncserver.go index 510546909c..3f1ca355e3 100644 --- a/syncapi/storage/sqlite3/syncserver.go +++ b/syncapi/storage/sqlite3/syncserver.go @@ -20,7 +20,6 @@ import ( "database/sql" "github.com/matrix-org/dendrite/internal/sqlutil" - "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/syncapi/storage/shared" "github.com/matrix-org/dendrite/syncapi/storage/sqlite3/deltas" @@ -37,13 +36,14 @@ type SyncServerDatasource struct { // NewDatabase creates a new sync server database // nolint: gocyclo -func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions) (*SyncServerDatasource, error) { +func NewDatabase(ctx context.Context, conMan sqlutil.Connections, dbProperties *config.DatabaseOptions) (*SyncServerDatasource, error) { var d SyncServerDatasource var err error - if d.db, d.writer, err = base.DatabaseConnection(dbProperties, sqlutil.NewExclusiveWriter()); err != nil { + + if d.db, d.writer, err = conMan.Connection(dbProperties); err != nil { return nil, err } - if err = d.prepare(base.Context()); err != nil { + if err = d.prepare(ctx); err != nil { return nil, err } return &d, nil diff --git a/syncapi/storage/storage.go b/syncapi/storage/storage.go index 5b20c6cc27..8714ec5e24 100644 --- a/syncapi/storage/storage.go +++ b/syncapi/storage/storage.go @@ -18,21 +18,22 @@ package storage import ( + "context" "fmt" - "github.com/matrix-org/dendrite/setup/base" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/syncapi/storage/postgres" "github.com/matrix-org/dendrite/syncapi/storage/sqlite3" ) // NewSyncServerDatasource opens a database connection. -func NewSyncServerDatasource(base *base.BaseDendrite, dbProperties *config.DatabaseOptions) (Database, error) { +func NewSyncServerDatasource(ctx context.Context, conMan sqlutil.Connections, dbProperties *config.DatabaseOptions) (Database, error) { switch { case dbProperties.ConnectionString.IsSQLite(): - return sqlite3.NewDatabase(base, dbProperties) + return sqlite3.NewDatabase(ctx, conMan, dbProperties) case dbProperties.ConnectionString.IsPostgres(): - return postgres.NewDatabase(base, dbProperties) + return postgres.NewDatabase(ctx, conMan, dbProperties) default: return nil, fmt.Errorf("unexpected database type") } diff --git a/syncapi/storage/storage_test.go b/syncapi/storage/storage_test.go index 05d498bc20..9f0064907b 100644 --- a/syncapi/storage/storage_test.go +++ b/syncapi/storage/storage_test.go @@ -9,27 +9,27 @@ import ( "reflect" "testing" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/syncapi/storage" "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/dendrite/test" - "github.com/matrix-org/dendrite/test/testrig" "github.com/matrix-org/gomatrixserverlib" "github.com/stretchr/testify/assert" ) var ctx = context.Background() -func MustCreateDatabase(t *testing.T, dbType test.DBType) (storage.Database, func(), func()) { +func MustCreateDatabase(t *testing.T, dbType test.DBType) (storage.Database, func()) { connStr, close := test.PrepareDBConnectionString(t, dbType) - base, closeBase := testrig.CreateBaseDendrite(t, dbType) - db, err := storage.NewSyncServerDatasource(base, &config.DatabaseOptions{ + cm := sqlutil.NewConnectionManager() + db, err := storage.NewSyncServerDatasource(context.Background(), cm, &config.DatabaseOptions{ ConnectionString: config.DataSource(connStr), }) if err != nil { t.Fatalf("NewSyncServerDatasource returned %s", err) } - return db, close, closeBase + return db, close } func MustWriteEvents(t *testing.T, db storage.Database, events []*gomatrixserverlib.HeaderedEvent) (positions []types.StreamPosition) { @@ -55,9 +55,8 @@ func TestWriteEvents(t *testing.T) { test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { alice := test.NewUser(t) r := test.NewRoom(t, alice) - db, close, closeBase := MustCreateDatabase(t, dbType) + db, close := MustCreateDatabase(t, dbType) defer close() - defer closeBase() MustWriteEvents(t, db, r.Events()) }) } @@ -76,9 +75,8 @@ func WithSnapshot(t *testing.T, db storage.Database, f func(snapshot storage.Dat // These tests assert basic functionality of RecentEvents for PDUs func TestRecentEventsPDU(t *testing.T) { test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { - db, close, closeBase := MustCreateDatabase(t, dbType) + db, close := MustCreateDatabase(t, dbType) defer close() - defer closeBase() alice := test.NewUser(t) // dummy room to make sure SQL queries are filtering on room ID MustWriteEvents(t, db, test.NewRoom(t, alice).Events()) @@ -191,9 +189,8 @@ func TestRecentEventsPDU(t *testing.T) { // The purpose of this test is to ensure that backfill does indeed go backwards, using a topology token func TestGetEventsInRangeWithTopologyToken(t *testing.T) { test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { - db, close, closeBase := MustCreateDatabase(t, dbType) + db, close := MustCreateDatabase(t, dbType) defer close() - defer closeBase() alice := test.NewUser(t) r := test.NewRoom(t, alice) for i := 0; i < 10; i++ { @@ -276,9 +273,8 @@ func TestStreamToTopologicalPosition(t *testing.T) { } test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { - db, close, closeBase := MustCreateDatabase(t, dbType) + db, close := MustCreateDatabase(t, dbType) defer close() - defer closeBase() txn, err := db.NewDatabaseTransaction(ctx) if err != nil { @@ -514,9 +510,8 @@ func TestSendToDeviceBehaviour(t *testing.T) { bob := test.NewUser(t) deviceID := "one" test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { - db, close, closeBase := MustCreateDatabase(t, dbType) + db, close := MustCreateDatabase(t, dbType) defer close() - defer closeBase() // At this point there should be no messages. We haven't sent anything // yet. @@ -899,9 +894,8 @@ func TestRoomSummary(t *testing.T) { } test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { - db, close, closeBase := MustCreateDatabase(t, dbType) + db, close := MustCreateDatabase(t, dbType) defer close() - defer closeBase() for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { @@ -939,11 +933,8 @@ func TestRecentEvents(t *testing.T) { test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { filter := gomatrixserverlib.DefaultRoomEventFilter() - db, close, closeBase := MustCreateDatabase(t, dbType) - t.Cleanup(func() { - close() - closeBase() - }) + db, close := MustCreateDatabase(t, dbType) + t.Cleanup(close) MustWriteEvents(t, db, room1.Events()) MustWriteEvents(t, db, room2.Events()) diff --git a/syncapi/storage/storage_wasm.go b/syncapi/storage/storage_wasm.go index c154447437..db0b173bbc 100644 --- a/syncapi/storage/storage_wasm.go +++ b/syncapi/storage/storage_wasm.go @@ -15,18 +15,19 @@ package storage import ( + "context" "fmt" - "github.com/matrix-org/dendrite/setup/base" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/syncapi/storage/sqlite3" ) // NewPublicRoomsServerDatabase opens a database connection. -func NewSyncServerDatasource(base *base.BaseDendrite, dbProperties *config.DatabaseOptions) (Database, error) { +func NewSyncServerDatasource(ctx context.Context, conMan sqlutil.Connections, dbProperties *config.DatabaseOptions) (Database, error) { switch { case dbProperties.ConnectionString.IsSQLite(): - return sqlite3.NewDatabase(base, dbProperties) + return sqlite3.NewDatabase(ctx, conMan, dbProperties) case dbProperties.ConnectionString.IsPostgres(): return nil, fmt.Errorf("can't use Postgres implementation") default: diff --git a/syncapi/syncapi.go b/syncapi/syncapi.go index 153f7af5db..e0cc8462e2 100644 --- a/syncapi/syncapi.go +++ b/syncapi/syncapi.go @@ -17,6 +17,7 @@ package syncapi import ( "context" + "github.com/matrix-org/dendrite/internal/fulltext" "github.com/sirupsen/logrus" "github.com/matrix-org/dendrite/internal/caching" @@ -41,24 +42,34 @@ func AddPublicRoutes( base *base.BaseDendrite, userAPI userapi.SyncUserAPI, rsAPI api.SyncRoomserverAPI, + caches caching.LazyLoadCache, ) { cfg := &base.Cfg.SyncAPI js, natsClient := base.NATS.Prepare(base.ProcessContext, &cfg.Matrix.JetStream) - syncDB, err := storage.NewSyncServerDatasource(base, &cfg.Database) + syncDB, err := storage.NewSyncServerDatasource(base.Context(), base.ConnectionManager, &cfg.Database) if err != nil { logrus.WithError(err).Panicf("failed to connect to sync db") } eduCache := caching.NewTypingCache() notifier := notifier.NewNotifier() - streams := streams.NewSyncStreamProviders(syncDB, userAPI, rsAPI, eduCache, base.Caches, notifier) + streams := streams.NewSyncStreamProviders(syncDB, userAPI, rsAPI, eduCache, caches, notifier) notifier.SetCurrentPosition(streams.Latest(context.Background())) if err = notifier.Load(context.Background(), syncDB); err != nil { logrus.WithError(err).Panicf("failed to load notifier ") } + var fts *fulltext.Search + if cfg.Fulltext.Enabled { + fts, err = fulltext.New(base.ProcessContext.Context(), cfg.Fulltext) + if err != nil { + logrus.WithError(err).Panicf("failed to create full text") + } + base.ProcessContext.ComponentStarted() + } + federationPresenceProducer := &producers.FederationAPIPresenceProducer{ Topic: cfg.Matrix.JetStream.Prefixed(jetstream.OutputPresenceEvent), JetStream: js, @@ -86,7 +97,7 @@ func AddPublicRoutes( roomConsumer := consumers.NewOutputRoomEventConsumer( base.ProcessContext, cfg, js, syncDB, notifier, streams.PDUStreamProvider, - streams.InviteStreamProvider, rsAPI, base.Fulltext, + streams.InviteStreamProvider, rsAPI, fts, ) if err = roomConsumer.Start(); err != nil { logrus.WithError(err).Panicf("failed to start room server consumer") @@ -94,7 +105,7 @@ func AddPublicRoutes( clientConsumer := consumers.NewOutputClientDataConsumer( base.ProcessContext, cfg, js, natsClient, syncDB, notifier, - streams.AccountDataStreamProvider, base.Fulltext, + streams.AccountDataStreamProvider, fts, ) if err = clientConsumer.Start(); err != nil { logrus.WithError(err).Panicf("failed to start client data consumer") @@ -129,7 +140,7 @@ func AddPublicRoutes( } routing.Setup( - base.PublicClientAPIMux, requestPool, syncDB, userAPI, - rsAPI, cfg, base.Caches, base.Fulltext, + base.Routers.Client, requestPool, syncDB, userAPI, + rsAPI, cfg, caches, fts, ) } diff --git a/syncapi/syncapi_test.go b/syncapi/syncapi_test.go index 1cb82ce11f..13a078659c 100644 --- a/syncapi/syncapi_test.go +++ b/syncapi/syncapi_test.go @@ -10,6 +10,7 @@ import ( "testing" "time" + "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/gomatrixserverlib" "github.com/nats-io/nats.go" "github.com/tidwall/gjson" @@ -114,12 +115,13 @@ func testSyncAccessTokens(t *testing.T, dbType test.DBType) { } base, close := testrig.CreateBaseDendrite(t, dbType) + caches := caching.NewRistrettoCache(base.Cfg.Global.Cache.EstimatedMaxSize, base.Cfg.Global.Cache.MaxAge, caching.DisableMetrics) defer close() jsctx, _ := base.NATS.Prepare(base.ProcessContext, &base.Cfg.Global.JetStream) defer jetstream.DeleteAllStreams(jsctx, &base.Cfg.Global.JetStream) msgs := toNATSMsgs(t, base, room.Events()...) - AddPublicRoutes(base, &syncUserAPI{accounts: []userapi.Device{alice}}, &syncRoomserverAPI{rooms: []*test.Room{room}}) + AddPublicRoutes(base, &syncUserAPI{accounts: []userapi.Device{alice}}, &syncRoomserverAPI{rooms: []*test.Room{room}}, caches) testrig.MustPublishMsgs(t, jsctx, msgs...) testCases := []struct { @@ -162,7 +164,7 @@ func testSyncAccessTokens(t *testing.T, dbType test.DBType) { for _, tc := range testCases { w := httptest.NewRecorder() - base.PublicClientAPIMux.ServeHTTP(w, tc.req) + base.Routers.Client.ServeHTTP(w, tc.req) if w.Code != tc.wantCode { t.Fatalf("%s: got HTTP %d want %d", tc.name, w.Code, tc.wantCode) } @@ -218,12 +220,13 @@ func testSyncAPICreateRoomSyncEarly(t *testing.T, dbType test.DBType) { // m.room.history_visibility msgs := toNATSMsgs(t, base, room.Events()...) sinceTokens := make([]string, len(msgs)) - AddPublicRoutes(base, &syncUserAPI{accounts: []userapi.Device{alice}}, &syncRoomserverAPI{rooms: []*test.Room{room}}) + caches := caching.NewRistrettoCache(base.Cfg.Global.Cache.EstimatedMaxSize, base.Cfg.Global.Cache.MaxAge, caching.DisableMetrics) + AddPublicRoutes(base, &syncUserAPI{accounts: []userapi.Device{alice}}, &syncRoomserverAPI{rooms: []*test.Room{room}}, caches) for i, msg := range msgs { testrig.MustPublishMsgs(t, jsctx, msg) time.Sleep(100 * time.Millisecond) w := httptest.NewRecorder() - base.PublicClientAPIMux.ServeHTTP(w, test.NewRequest(t, "GET", "/_matrix/client/v3/sync", test.WithQueryParams(map[string]string{ + base.Routers.Client.ServeHTTP(w, test.NewRequest(t, "GET", "/_matrix/client/v3/sync", test.WithQueryParams(map[string]string{ "access_token": alice.AccessToken, "timeout": "0", }))) @@ -253,7 +256,7 @@ func testSyncAPICreateRoomSyncEarly(t *testing.T, dbType test.DBType) { t.Logf("waited for events to be consumed; syncing with %v", sinceTokens) for i, since := range sinceTokens { w := httptest.NewRecorder() - base.PublicClientAPIMux.ServeHTTP(w, test.NewRequest(t, "GET", "/_matrix/client/v3/sync", test.WithQueryParams(map[string]string{ + base.Routers.Client.ServeHTTP(w, test.NewRequest(t, "GET", "/_matrix/client/v3/sync", test.WithQueryParams(map[string]string{ "access_token": alice.AccessToken, "timeout": "0", "since": since, @@ -302,9 +305,10 @@ func testSyncAPIUpdatePresenceImmediately(t *testing.T, dbType test.DBType) { jsctx, _ := base.NATS.Prepare(base.ProcessContext, &base.Cfg.Global.JetStream) defer jetstream.DeleteAllStreams(jsctx, &base.Cfg.Global.JetStream) - AddPublicRoutes(base, &syncUserAPI{accounts: []userapi.Device{alice}}, &syncRoomserverAPI{}) + caches := caching.NewRistrettoCache(base.Cfg.Global.Cache.EstimatedMaxSize, base.Cfg.Global.Cache.MaxAge, caching.DisableMetrics) + AddPublicRoutes(base, &syncUserAPI{accounts: []userapi.Device{alice}}, &syncRoomserverAPI{}, caches) w := httptest.NewRecorder() - base.PublicClientAPIMux.ServeHTTP(w, test.NewRequest(t, "GET", "/_matrix/client/v3/sync", test.WithQueryParams(map[string]string{ + base.Routers.Client.ServeHTTP(w, test.NewRequest(t, "GET", "/_matrix/client/v3/sync", test.WithQueryParams(map[string]string{ "access_token": alice.AccessToken, "timeout": "0", "set_presence": "online", @@ -417,10 +421,10 @@ func testHistoryVisibility(t *testing.T, dbType test.DBType) { defer jetstream.DeleteAllStreams(jsctx, &base.Cfg.Global.JetStream) // Use the actual internal roomserver API - rsAPI := roomserver.NewInternalAPI(base) + caches := caching.NewRistrettoCache(base.Cfg.Global.Cache.EstimatedMaxSize, base.Cfg.Global.Cache.MaxAge, caching.DisableMetrics) + rsAPI := roomserver.NewInternalAPI(base, caches) rsAPI.SetFederationAPI(nil, nil) - - AddPublicRoutes(base, &syncUserAPI{accounts: []userapi.Device{aliceDev, bobDev}}, rsAPI) + AddPublicRoutes(base, &syncUserAPI{accounts: []userapi.Device{aliceDev, bobDev}}, rsAPI, caches) for _, tc := range testCases { testname := fmt.Sprintf("%s - %s", tc.historyVisibility, userType) @@ -444,7 +448,7 @@ func testHistoryVisibility(t *testing.T, dbType test.DBType) { // There is only one event, we expect only to be able to see this, if the room is world_readable w := httptest.NewRecorder() - base.PublicClientAPIMux.ServeHTTP(w, test.NewRequest(t, "GET", fmt.Sprintf("/_matrix/client/v3/rooms/%s/messages", room.ID), test.WithQueryParams(map[string]string{ + base.Routers.Client.ServeHTTP(w, test.NewRequest(t, "GET", fmt.Sprintf("/_matrix/client/v3/rooms/%s/messages", room.ID), test.WithQueryParams(map[string]string{ "access_token": bobDev.AccessToken, "dir": "b", "filter": `{"lazy_load_members":true}`, // check that lazy loading doesn't break history visibility @@ -484,7 +488,7 @@ func testHistoryVisibility(t *testing.T, dbType test.DBType) { // Verify the messages after/before invite are visible or not w = httptest.NewRecorder() - base.PublicClientAPIMux.ServeHTTP(w, test.NewRequest(t, "GET", fmt.Sprintf("/_matrix/client/v3/rooms/%s/messages", room.ID), test.WithQueryParams(map[string]string{ + base.Routers.Client.ServeHTTP(w, test.NewRequest(t, "GET", fmt.Sprintf("/_matrix/client/v3/rooms/%s/messages", room.ID), test.WithQueryParams(map[string]string{ "access_token": bobDev.AccessToken, "dir": "b", }))) @@ -717,10 +721,11 @@ func TestGetMembership(t *testing.T) { defer jetstream.DeleteAllStreams(jsctx, &base.Cfg.Global.JetStream) // Use an actual roomserver for this - rsAPI := roomserver.NewInternalAPI(base) + caches := caching.NewRistrettoCache(base.Cfg.Global.Cache.EstimatedMaxSize, base.Cfg.Global.Cache.MaxAge, caching.DisableMetrics) + rsAPI := roomserver.NewInternalAPI(base, caches) rsAPI.SetFederationAPI(nil, nil) - AddPublicRoutes(base, &syncUserAPI{accounts: []userapi.Device{aliceDev, bobDev}}, rsAPI) + AddPublicRoutes(base, &syncUserAPI{accounts: []userapi.Device{aliceDev, bobDev}}, rsAPI, caches) for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { @@ -748,7 +753,7 @@ func TestGetMembership(t *testing.T) { } w := httptest.NewRecorder() - base.PublicClientAPIMux.ServeHTTP(w, tc.request(t, room)) + base.Routers.Client.ServeHTTP(w, tc.request(t, room)) if w.Code != 200 && tc.wantOK { t.Logf("%s", w.Body.String()) t.Fatalf("got HTTP %d want %d", w.Code, 200) @@ -786,8 +791,8 @@ func testSendToDevice(t *testing.T, dbType test.DBType) { jsctx, _ := base.NATS.Prepare(base.ProcessContext, &base.Cfg.Global.JetStream) defer jetstream.DeleteAllStreams(jsctx, &base.Cfg.Global.JetStream) - - AddPublicRoutes(base, &syncUserAPI{accounts: []userapi.Device{alice}}, &syncRoomserverAPI{}) + caches := caching.NewRistrettoCache(base.Cfg.Global.Cache.EstimatedMaxSize, base.Cfg.Global.Cache.MaxAge, caching.DisableMetrics) + AddPublicRoutes(base, &syncUserAPI{accounts: []userapi.Device{alice}}, &syncRoomserverAPI{}, caches) producer := producers.SyncAPIProducer{ TopicSendToDeviceEvent: base.Cfg.Global.JetStream.Prefixed(jetstream.OutputSendToDeviceEvent), @@ -885,7 +890,7 @@ func testSendToDevice(t *testing.T, dbType test.DBType) { // Execute a /sync request, recording the response w := httptest.NewRecorder() - base.PublicClientAPIMux.ServeHTTP(w, test.NewRequest(t, "GET", "/_matrix/client/v3/sync", test.WithQueryParams(map[string]string{ + base.Routers.Client.ServeHTTP(w, test.NewRequest(t, "GET", "/_matrix/client/v3/sync", test.WithQueryParams(map[string]string{ "access_token": alice.AccessToken, "since": tc.since, }))) @@ -1003,10 +1008,11 @@ func testContext(t *testing.T, dbType test.DBType) { defer baseClose() // Use an actual roomserver for this - rsAPI := roomserver.NewInternalAPI(base) + caches := caching.NewRistrettoCache(base.Cfg.Global.Cache.EstimatedMaxSize, base.Cfg.Global.Cache.MaxAge, caching.DisableMetrics) + rsAPI := roomserver.NewInternalAPI(base, caches) rsAPI.SetFederationAPI(nil, nil) - AddPublicRoutes(base, &syncUserAPI{accounts: []userapi.Device{alice}}, rsAPI) + AddPublicRoutes(base, &syncUserAPI{accounts: []userapi.Device{alice}}, rsAPI, caches) room := test.NewRoom(t, user) @@ -1049,7 +1055,7 @@ func testContext(t *testing.T, dbType test.DBType) { params[k] = v } } - base.PublicClientAPIMux.ServeHTTP(w, test.NewRequest(t, "GET", requestPath, test.WithQueryParams(params))) + base.Routers.Client.ServeHTTP(w, test.NewRequest(t, "GET", requestPath, test.WithQueryParams(params))) if tc.wantError && w.Code == 200 { t.Fatalf("Expected an error, but got none") @@ -1139,7 +1145,7 @@ func TestUpdateRelations(t *testing.T) { test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { base, shutdownBase := testrig.CreateBaseDendrite(t, dbType) t.Cleanup(shutdownBase) - db, err := storage.NewSyncServerDatasource(base, &base.Cfg.SyncAPI.Database) + db, err := storage.NewSyncServerDatasource(base.Context(), base.ConnectionManager, &base.Cfg.SyncAPI.Database) if err != nil { t.Fatal(err) } @@ -1178,7 +1184,7 @@ func syncUntil(t *testing.T, go func() { for { w := httptest.NewRecorder() - base.PublicClientAPIMux.ServeHTTP(w, test.NewRequest(t, "GET", "/_matrix/client/v3/sync", test.WithQueryParams(map[string]string{ + base.Routers.Client.ServeHTTP(w, test.NewRequest(t, "GET", "/_matrix/client/v3/sync", test.WithQueryParams(map[string]string{ "access_token": accessToken, "timeout": "1000", }))) diff --git a/userapi/consumers/roomserver_test.go b/userapi/consumers/roomserver_test.go index bc5ae652d8..a54706dbbc 100644 --- a/userapi/consumers/roomserver_test.go +++ b/userapi/consumers/roomserver_test.go @@ -7,22 +7,22 @@ import ( "testing" "time" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" "github.com/stretchr/testify/assert" "github.com/matrix-org/dendrite/internal/pushrules" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/test" - "github.com/matrix-org/dendrite/test/testrig" "github.com/matrix-org/dendrite/userapi/storage" userAPITypes "github.com/matrix-org/dendrite/userapi/types" ) func mustCreateDatabase(t *testing.T, dbType test.DBType) (storage.UserDatabase, func()) { - base, baseclose := testrig.CreateBaseDendrite(t, dbType) t.Helper() connStr, close := test.PrepareDBConnectionString(t, dbType) - db, err := storage.NewUserDatabase(base, &config.DatabaseOptions{ + cm := sqlutil.NewConnectionManager() + db, err := storage.NewUserDatabase(context.Background(), cm, &config.DatabaseOptions{ ConnectionString: config.DataSource(connStr), }, "", 4, 0, 0, "") if err != nil { @@ -30,7 +30,6 @@ func mustCreateDatabase(t *testing.T, dbType test.DBType) (storage.UserDatabase, } return db, func() { close() - baseclose() } } diff --git a/userapi/internal/device_list_update_test.go b/userapi/internal/device_list_update_test.go index 868fc9be8f..6ccc4b5fbc 100644 --- a/userapi/internal/device_list_update_test.go +++ b/userapi/internal/device_list_update_test.go @@ -27,13 +27,13 @@ import ( "testing" "time" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" roomserver "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/process" "github.com/matrix-org/dendrite/test" - "github.com/matrix-org/dendrite/test/testrig" "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/dendrite/userapi/storage" ) @@ -363,9 +363,9 @@ func TestDebounce(t *testing.T) { func mustCreateKeyserverDB(t *testing.T, dbType test.DBType) (storage.KeyDatabase, func()) { t.Helper() - base, _, _ := testrig.Base(nil) connStr, clearDB := test.PrepareDBConnectionString(t, dbType) - db, err := storage.NewKeyDatabase(base, &config.DatabaseOptions{ConnectionString: config.DataSource(connStr)}) + cm := sqlutil.NewConnectionManager() + db, err := storage.NewKeyDatabase(cm, &config.DatabaseOptions{ConnectionString: config.DataSource(connStr)}) if err != nil { t.Fatal(err) } diff --git a/userapi/internal/key_api_test.go b/userapi/internal/key_api_test.go index fc7e7e0dfa..ec315c738b 100644 --- a/userapi/internal/key_api_test.go +++ b/userapi/internal/key_api_test.go @@ -5,9 +5,9 @@ import ( "reflect" "testing" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/test" - "github.com/matrix-org/dendrite/test/testrig" "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/dendrite/userapi/internal" "github.com/matrix-org/dendrite/userapi/storage" @@ -16,15 +16,14 @@ import ( func mustCreateDatabase(t *testing.T, dbType test.DBType) (storage.KeyDatabase, func()) { t.Helper() connStr, close := test.PrepareDBConnectionString(t, dbType) - base, _, _ := testrig.Base(nil) - db, err := storage.NewKeyDatabase(base, &config.DatabaseOptions{ + cm := sqlutil.NewConnectionManager() + db, err := storage.NewKeyDatabase(cm, &config.DatabaseOptions{ ConnectionString: config.DataSource(connStr), }) if err != nil { t.Fatalf("failed to create new user db: %v", err) } return db, func() { - base.Close() close() } } diff --git a/userapi/storage/postgres/storage.go b/userapi/storage/postgres/storage.go index 673d123bab..7bfae7b209 100644 --- a/userapi/storage/postgres/storage.go +++ b/userapi/storage/postgres/storage.go @@ -23,7 +23,6 @@ import ( "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/dendrite/internal/sqlutil" - "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/userapi/storage/postgres/deltas" "github.com/matrix-org/dendrite/userapi/storage/shared" @@ -33,8 +32,8 @@ import ( ) // NewDatabase creates a new accounts and profiles database -func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions, serverName gomatrixserverlib.ServerName, bcryptCost int, openIDTokenLifetimeMS int64, loginTokenLifetime time.Duration, serverNoticesLocalpart string) (*shared.Database, error) { - db, writer, err := base.DatabaseConnection(dbProperties, sqlutil.NewDummyWriter()) +func NewDatabase(ctx context.Context, conMan sqlutil.Connections, dbProperties *config.DatabaseOptions, serverName gomatrixserverlib.ServerName, bcryptCost int, openIDTokenLifetimeMS int64, loginTokenLifetime time.Duration, serverNoticesLocalpart string) (*shared.Database, error) { + db, writer, err := conMan.Connection(dbProperties) if err != nil { return nil, err } @@ -51,7 +50,7 @@ func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions, return deltas.UpServerNames(ctx, txn, serverName) }, }) - if err = m.Up(base.Context()); err != nil { + if err = m.Up(ctx); err != nil { return nil, err } @@ -111,7 +110,7 @@ func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions, return deltas.UpServerNamesPopulate(ctx, txn, serverName) }, }) - if err = m.Up(base.Context()); err != nil { + if err = m.Up(ctx); err != nil { return nil, err } @@ -137,8 +136,8 @@ func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions, }, nil } -func NewKeyDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions) (*shared.KeyDatabase, error) { - db, writer, err := base.DatabaseConnection(dbProperties, sqlutil.NewDummyWriter()) +func NewKeyDatabase(conMan sqlutil.Connections, dbProperties *config.DatabaseOptions) (*shared.KeyDatabase, error) { + db, writer, err := conMan.Connection(dbProperties) if err != nil { return nil, err } diff --git a/userapi/storage/sqlite3/storage.go b/userapi/storage/sqlite3/storage.go index 0f3eeed1b1..3742eebada 100644 --- a/userapi/storage/sqlite3/storage.go +++ b/userapi/storage/sqlite3/storage.go @@ -23,7 +23,6 @@ import ( "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/dendrite/internal/sqlutil" - "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/userapi/storage/shared" @@ -31,8 +30,8 @@ import ( ) // NewUserDatabase creates a new accounts and profiles database -func NewUserDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions, serverName gomatrixserverlib.ServerName, bcryptCost int, openIDTokenLifetimeMS int64, loginTokenLifetime time.Duration, serverNoticesLocalpart string) (*shared.Database, error) { - db, writer, err := base.DatabaseConnection(dbProperties, sqlutil.NewExclusiveWriter()) +func NewUserDatabase(ctx context.Context, conMan sqlutil.Connections, dbProperties *config.DatabaseOptions, serverName gomatrixserverlib.ServerName, bcryptCost int, openIDTokenLifetimeMS int64, loginTokenLifetime time.Duration, serverNoticesLocalpart string) (*shared.Database, error) { + db, writer, err := conMan.Connection(dbProperties) if err != nil { return nil, err } @@ -49,7 +48,7 @@ func NewUserDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptio return deltas.UpServerNames(ctx, txn, serverName) }, }) - if err = m.Up(base.Context()); err != nil { + if err = m.Up(ctx); err != nil { return nil, err } @@ -109,7 +108,7 @@ func NewUserDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptio return deltas.UpServerNamesPopulate(ctx, txn, serverName) }, }) - if err = m.Up(base.Context()); err != nil { + if err = m.Up(ctx); err != nil { return nil, err } @@ -135,8 +134,8 @@ func NewUserDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptio }, nil } -func NewKeyDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions) (*shared.KeyDatabase, error) { - db, writer, err := base.DatabaseConnection(dbProperties, sqlutil.NewExclusiveWriter()) +func NewKeyDatabase(conMan sqlutil.Connections, dbProperties *config.DatabaseOptions) (*shared.KeyDatabase, error) { + db, writer, err := conMan.Connection(dbProperties) if err != nil { return nil, err } diff --git a/userapi/storage/storage.go b/userapi/storage/storage.go index 0329fb46a4..6981765f99 100644 --- a/userapi/storage/storage.go +++ b/userapi/storage/storage.go @@ -18,12 +18,13 @@ package storage import ( + "context" "fmt" "time" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" - "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/userapi/storage/postgres" "github.com/matrix-org/dendrite/userapi/storage/sqlite3" @@ -32,7 +33,8 @@ import ( // NewUserDatabase opens a new Postgres or Sqlite database (based on dataSourceName scheme) // and sets postgres connection parameters func NewUserDatabase( - base *base.BaseDendrite, + ctx context.Context, + conMan sqlutil.Connections, dbProperties *config.DatabaseOptions, serverName gomatrixserverlib.ServerName, bcryptCost int, @@ -42,9 +44,9 @@ func NewUserDatabase( ) (UserDatabase, error) { switch { case dbProperties.ConnectionString.IsSQLite(): - return sqlite3.NewUserDatabase(base, dbProperties, serverName, bcryptCost, openIDTokenLifetimeMS, loginTokenLifetime, serverNoticesLocalpart) + return sqlite3.NewUserDatabase(ctx, conMan, dbProperties, serverName, bcryptCost, openIDTokenLifetimeMS, loginTokenLifetime, serverNoticesLocalpart) case dbProperties.ConnectionString.IsPostgres(): - return postgres.NewDatabase(base, dbProperties, serverName, bcryptCost, openIDTokenLifetimeMS, loginTokenLifetime, serverNoticesLocalpart) + return postgres.NewDatabase(ctx, conMan, dbProperties, serverName, bcryptCost, openIDTokenLifetimeMS, loginTokenLifetime, serverNoticesLocalpart) default: return nil, fmt.Errorf("unexpected database type") } @@ -52,12 +54,12 @@ func NewUserDatabase( // NewKeyDatabase opens a new Postgres or Sqlite database (base on dataSourceName) scheme) // and sets postgres connection parameters. -func NewKeyDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions) (KeyDatabase, error) { +func NewKeyDatabase(conMan sqlutil.Connections, dbProperties *config.DatabaseOptions) (KeyDatabase, error) { switch { case dbProperties.ConnectionString.IsSQLite(): - return sqlite3.NewKeyDatabase(base, dbProperties) + return sqlite3.NewKeyDatabase(conMan, dbProperties) case dbProperties.ConnectionString.IsPostgres(): - return postgres.NewKeyDatabase(base, dbProperties) + return postgres.NewKeyDatabase(conMan, dbProperties) default: return nil, fmt.Errorf("unexpected database type") } diff --git a/userapi/storage/storage_test.go b/userapi/storage/storage_test.go index f52e7e17db..c6369ef932 100644 --- a/userapi/storage/storage_test.go +++ b/userapi/storage/storage_test.go @@ -9,6 +9,7 @@ import ( "testing" "time" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/userapi/types" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" @@ -33,9 +34,9 @@ var ( ) func mustCreateUserDatabase(t *testing.T, dbType test.DBType) (storage.UserDatabase, func()) { - base, baseclose := testrig.CreateBaseDendrite(t, dbType) connStr, close := test.PrepareDBConnectionString(t, dbType) - db, err := storage.NewUserDatabase(base, &config.DatabaseOptions{ + cm := sqlutil.NewConnectionManager() + db, err := storage.NewUserDatabase(context.Background(), cm, &config.DatabaseOptions{ ConnectionString: config.DataSource(connStr), }, "localhost", bcrypt.MinCost, openIDLifetimeMS, loginTokenLifetime, "_server") if err != nil { @@ -43,7 +44,6 @@ func mustCreateUserDatabase(t *testing.T, dbType test.DBType) (storage.UserDatab } return db, func() { close() - baseclose() } } @@ -577,7 +577,7 @@ func Test_Notification(t *testing.T) { func mustCreateKeyDatabase(t *testing.T, dbType test.DBType) (storage.KeyDatabase, func()) { base, close := testrig.CreateBaseDendrite(t, dbType) - db, err := storage.NewKeyDatabase(base, &base.Cfg.KeyServer.Database) + db, err := storage.NewKeyDatabase(base.ConnectionManager, &base.Cfg.KeyServer.Database) if err != nil { t.Fatalf("failed to create new database: %v", err) } diff --git a/userapi/storage/storage_wasm.go b/userapi/storage/storage_wasm.go index 163e3e1731..19e5f23c63 100644 --- a/userapi/storage/storage_wasm.go +++ b/userapi/storage/storage_wasm.go @@ -15,17 +15,19 @@ package storage import ( + "context" "fmt" "time" - "github.com/matrix-org/dendrite/setup/base" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/userapi/storage/sqlite3" "github.com/matrix-org/gomatrixserverlib" ) -func NewUserAPIDatabase( - base *base.BaseDendrite, +func NewUserDatabase( + ctx context.Context, + conMan sqlutil.Connections, dbProperties *config.DatabaseOptions, serverName gomatrixserverlib.ServerName, bcryptCost int, @@ -35,7 +37,20 @@ func NewUserAPIDatabase( ) (UserDatabase, error) { switch { case dbProperties.ConnectionString.IsSQLite(): - return sqlite3.NewUserDatabase(base, dbProperties, serverName, bcryptCost, openIDTokenLifetimeMS, loginTokenLifetime, serverNoticesLocalpart) + return sqlite3.NewUserDatabase(ctx, conMan, dbProperties, serverName, bcryptCost, openIDTokenLifetimeMS, loginTokenLifetime, serverNoticesLocalpart) + case dbProperties.ConnectionString.IsPostgres(): + return nil, fmt.Errorf("can't use Postgres implementation") + default: + return nil, fmt.Errorf("unexpected database type") + } +} + +// NewKeyDatabase opens a new Postgres or Sqlite database (base on dataSourceName) scheme) +// and sets postgres connection parameters. +func NewKeyDatabase(conMan sqlutil.Connections, dbProperties *config.DatabaseOptions) (KeyDatabase, error) { + switch { + case dbProperties.ConnectionString.IsSQLite(): + return sqlite3.NewKeyDatabase(conMan, dbProperties) case dbProperties.ConnectionString.IsPostgres(): return nil, fmt.Errorf("can't use Postgres implementation") default: diff --git a/userapi/userapi.go b/userapi/userapi.go index 826bd72138..3ada8020c6 100644 --- a/userapi/userapi.go +++ b/userapi/userapi.go @@ -18,6 +18,7 @@ import ( "time" fedsenderapi "github.com/matrix-org/dendrite/federationapi/api" + "github.com/matrix-org/dendrite/internal/pushgateway" "github.com/sirupsen/logrus" rsapi "github.com/matrix-org/dendrite/roomserver/api" @@ -42,10 +43,11 @@ func NewInternalAPI( js, _ := base.NATS.Prepare(base.ProcessContext, &cfg.Matrix.JetStream) appServices := base.Cfg.Derived.ApplicationServices - pgClient := base.PushGatewayHTTPClient() + pgClient := pushgateway.NewHTTPClient(cfg.PushGatewayDisableTLSValidation) db, err := storage.NewUserDatabase( - base, + base.ProcessContext.Context(), + base.ConnectionManager, &cfg.AccountDatabase, cfg.Matrix.ServerName, cfg.BCryptCost, @@ -57,7 +59,7 @@ func NewInternalAPI( logrus.WithError(err).Panicf("failed to connect to accounts db") } - keyDB, err := storage.NewKeyDatabase(base, &base.Cfg.KeyServer.Database) + keyDB, err := storage.NewKeyDatabase(base.ConnectionManager, &base.Cfg.KeyServer.Database) if err != nil { logrus.WithError(err).Panicf("failed to connect to key db") } diff --git a/userapi/userapi_test.go b/userapi/userapi_test.go index 01e491cb6e..c2d4e5a270 100644 --- a/userapi/userapi_test.go +++ b/userapi/userapi_test.go @@ -22,6 +22,7 @@ import ( "testing" "time" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/userapi/producers" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" @@ -73,14 +74,16 @@ func MustMakeInternalAPI(t *testing.T, opts apiTestOpts, dbType test.DBType, pub if opts.serverName != "" { sName = gomatrixserverlib.ServerName(opts.serverName) } - accountDB, err := storage.NewUserDatabase(base, &config.DatabaseOptions{ + cm := sqlutil.NewConnectionManager() + ctx := context.Background() + accountDB, err := storage.NewUserDatabase(ctx, cm, &config.DatabaseOptions{ ConnectionString: config.DataSource(connStr), }, sName, bcrypt.MinCost, config.DefaultOpenIDTokenLifetimeMS, opts.loginTokenLifetime, "") if err != nil { t.Fatalf("failed to create account DB: %s", err) } - keyDB, err := storage.NewKeyDatabase(base, &config.DatabaseOptions{ + keyDB, err := storage.NewKeyDatabase(base.ConnectionManager, &config.DatabaseOptions{ ConnectionString: config.DataSource(connStr), }) if err != nil { diff --git a/userapi/util/notify_test.go b/userapi/util/notify_test.go index 421852d3f8..c899e3a7c0 100644 --- a/userapi/util/notify_test.go +++ b/userapi/util/notify_test.go @@ -8,6 +8,7 @@ import ( "testing" "time" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" "golang.org/x/crypto/bcrypt" @@ -15,7 +16,6 @@ import ( "github.com/matrix-org/dendrite/internal/pushgateway" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/test" - "github.com/matrix-org/dendrite/test/testrig" "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/dendrite/userapi/storage" userUtil "github.com/matrix-org/dendrite/userapi/util" @@ -77,9 +77,8 @@ func TestNotifyUserCountsAsync(t *testing.T) { // Create DB and Dendrite base connStr, close := test.PrepareDBConnectionString(t, dbType) defer close() - base, _, _ := testrig.Base(nil) - defer base.Close() - db, err := storage.NewUserDatabase(base, &config.DatabaseOptions{ + cm := sqlutil.NewConnectionManager() + db, err := storage.NewUserDatabase(ctx, cm, &config.DatabaseOptions{ ConnectionString: config.DataSource(connStr), }, "test", bcrypt.MinCost, 0, 0, "") if err != nil { diff --git a/userapi/util/phonehomestats_test.go b/userapi/util/phonehomestats_test.go index 5f626b5bcb..4e931a1b7f 100644 --- a/userapi/util/phonehomestats_test.go +++ b/userapi/util/phonehomestats_test.go @@ -21,7 +21,7 @@ func TestCollect(t *testing.T) { b, _, _ := testrig.Base(nil) connStr, closeDB := test.PrepareDBConnectionString(t, dbType) defer closeDB() - db, err := storage.NewUserDatabase(b, &config.DatabaseOptions{ + db, err := storage.NewUserDatabase(b.Context(), b.ConnectionManager, &config.DatabaseOptions{ ConnectionString: config.DataSource(connStr), }, "localhost", bcrypt.MinCost, 1000, 1000, "") if err != nil { From 0459d2b9e586b53d0ffc9f362f1dcd42b79995f5 Mon Sep 17 00:00:00 2001 From: Till Faelligen <2353100+S7evinK@users.noreply.github.com> Date: Mon, 20 Mar 2023 09:24:00 +0100 Subject: [PATCH 04/42] Make "m.upload.size" optional --- mediaapi/routing/routing.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mediaapi/routing/routing.go b/mediaapi/routing/routing.go index 50af2f884d..8a4a5afcb7 100644 --- a/mediaapi/routing/routing.go +++ b/mediaapi/routing/routing.go @@ -35,7 +35,7 @@ import ( // configResponse is the response to GET /_matrix/media/r0/config // https://matrix.org/docs/spec/client_server/latest#get-matrix-media-r0-config type configResponse struct { - UploadSize *config.FileSizeBytes `json:"m.upload.size"` + UploadSize *config.FileSizeBytes `json:"m.upload.size,omitempty"` } // Setup registers the media API HTTP handlers From ec6879e5ae2919c903707475ce8d72244b2a6847 Mon Sep 17 00:00:00 2001 From: Till Faelligen <2353100+S7evinK@users.noreply.github.com> Date: Tue, 21 Mar 2023 16:04:51 +0100 Subject: [PATCH 05/42] Update GMSL to fix #3013 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 780867a1e6..cd9e0838fa 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530 - github.com/matrix-org/gomatrixserverlib v0.0.0-20230131183213-122f1e0e3fa1 + github.com/matrix-org/gomatrixserverlib v0.0.0-20230320105331-4dd7ff2f0e3a github.com/matrix-org/pinecone v0.11.1-0.20230210171230-8c3b24f2649a github.com/matrix-org/util v0.0.0-20221111132719-399730281e66 github.com/mattn/go-sqlite3 v1.14.16 diff --git a/go.sum b/go.sum index 1419b26e4f..1484f8aef2 100644 --- a/go.sum +++ b/go.sum @@ -321,8 +321,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 h1:s7fexw github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo= github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530 h1:kHKxCOLcHH8r4Fzarl4+Y3K5hjothkVW5z7T1dUM11U= github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= -github.com/matrix-org/gomatrixserverlib v0.0.0-20230131183213-122f1e0e3fa1 h1:JSw0nmjMrgBmoM2aQsa78LTpI5BnuD9+vOiEQ4Qo0qw= -github.com/matrix-org/gomatrixserverlib v0.0.0-20230131183213-122f1e0e3fa1/go.mod h1:Mtifyr8q8htcBeugvlDnkBcNUy5LO8OzUoplAf1+mb4= +github.com/matrix-org/gomatrixserverlib v0.0.0-20230320105331-4dd7ff2f0e3a h1:F6K1i61KcJ8cX/y0Q8/44Dh1w+fpESQd92gq885FDrI= +github.com/matrix-org/gomatrixserverlib v0.0.0-20230320105331-4dd7ff2f0e3a/go.mod h1:7HTbSZe+CIdmeqVyFMekwD5dFU8khWQyngKATvd12FU= github.com/matrix-org/pinecone v0.11.1-0.20230210171230-8c3b24f2649a h1:awrPDf9LEFySxTLKYBMCiObelNx/cBuv/wzllvCCH3A= github.com/matrix-org/pinecone v0.11.1-0.20230210171230-8c3b24f2649a/go.mod h1:HchJX9oKMXaT2xYFs0Ha/6Zs06mxLU8k6F1ODnrGkeQ= github.com/matrix-org/util v0.0.0-20221111132719-399730281e66 h1:6z4KxomXSIGWqhHcfzExgkH3Z3UkIXry4ibJS4Aqz2Y= From 5e85a00cb36c3d343cd5b6f6a18435989724a135 Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Wed, 22 Mar 2023 09:21:32 +0100 Subject: [PATCH 06/42] Remove `BaseDendrite` (#3023) Removes `BaseDendrite` to, hopefully, make testing and composing of components easier in the future. --- appservice/appservice.go | 21 +- appservice/appservice_test.go | 23 +- build/dendritejs-pinecone/main.go | 40 +-- build/gobind-pinecone/monolith.go | 13 +- build/gobind-yggdrasil/monolith.go | 89 +++++-- clientapi/admin_test.go | 50 ++-- clientapi/clientapi.go | 29 ++- clientapi/routing/auth_fallback_test.go | 46 ++-- clientapi/routing/joinroom_test.go | 20 +- clientapi/routing/login_test.go | 25 +- clientapi/routing/register_test.go | 74 +++--- clientapi/routing/routing.go | 23 +- cmd/dendrite-demo-pinecone/conn/client.go | 8 +- cmd/dendrite-demo-pinecone/main.go | 9 +- .../monolith/monolith.go | 81 +++--- cmd/dendrite-demo-yggdrasil/main.go | 94 +++++-- cmd/dendrite-demo-yggdrasil/yggconn/client.go | 10 +- cmd/dendrite-upgrade-tests/main.go | 16 +- cmd/dendrite/main.go | 111 ++++++-- cmd/resolve-state/main.go | 9 +- federationapi/federationapi.go | 51 ++-- federationapi/federationapi_keys_test.go | 11 +- federationapi/federationapi_test.go | 75 +++--- federationapi/queue/queue_test.go | 14 +- federationapi/routing/profile_test.go | 18 +- federationapi/routing/query_test.go | 18 +- federationapi/routing/routing.go | 16 +- federationapi/routing/send_test.go | 18 +- federationapi/storage/storage_test.go | 2 +- internal/sqlutil/connection_manager.go | 31 ++- internal/sqlutil/connection_manager_test.go | 8 +- mediaapi/mediaapi.go | 15 +- mediaapi/routing/routing.go | 15 +- mediaapi/routing/upload_test.go | 2 +- mediaapi/storage/storage_test.go | 2 +- relayapi/relayapi.go | 23 +- relayapi/relayapi_test.go | 65 +++-- roomserver/internal/alias.go | 8 +- roomserver/internal/api.go | 50 ++-- roomserver/internal/helpers/helpers_test.go | 2 +- roomserver/internal/input/input.go | 9 +- roomserver/internal/input/input_test.go | 117 ++++----- roomserver/roomserver.go | 19 +- roomserver/roomserver_test.go | 71 ++--- roomserver/storage/shared/storage_test.go | 4 - setup/base/base.go | 242 ++++-------------- setup/base/base_test.go | 20 +- setup/base/sanity_other.go | 2 +- setup/base/sanity_unix.go | 2 +- setup/jetstream/nats.go | 4 +- setup/monolith.go | 27 +- setup/mscs/msc2836/msc2836.go | 13 +- setup/mscs/msc2836/msc2836_test.go | 15 +- setup/mscs/msc2946/msc2946.go | 18 +- setup/mscs/mscs.go | 16 +- syncapi/storage/storage_test.go | 2 +- syncapi/syncapi.go | 48 ++-- syncapi/syncapi_test.go | 162 +++++++----- test/testrig/base.go | 43 +--- test/testrig/jetstream.go | 6 +- userapi/consumers/roomserver_test.go | 2 +- userapi/internal/device_list_update_test.go | 2 +- userapi/internal/key_api_test.go | 2 +- userapi/storage/storage_test.go | 7 +- userapi/userapi.go | 58 +++-- userapi/userapi_test.go | 26 +- userapi/util/notify_test.go | 2 +- userapi/util/phonehomestats_test.go | 14 +- 68 files changed, 1186 insertions(+), 1002 deletions(-) diff --git a/appservice/appservice.go b/appservice/appservice.go index 5b1b93de27..d13d9eb102 100644 --- a/appservice/appservice.go +++ b/appservice/appservice.go @@ -21,6 +21,8 @@ import ( "sync" "time" + "github.com/matrix-org/dendrite/setup/jetstream" + "github.com/matrix-org/dendrite/setup/process" "github.com/sirupsen/logrus" "github.com/matrix-org/gomatrixserverlib" @@ -29,7 +31,6 @@ import ( "github.com/matrix-org/dendrite/appservice/consumers" "github.com/matrix-org/dendrite/appservice/query" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" - "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/config" userapi "github.com/matrix-org/dendrite/userapi/api" ) @@ -37,7 +38,9 @@ import ( // NewInternalAPI returns a concerete implementation of the internal API. Callers // can call functions directly on the returned API or via an HTTP interface using AddInternalRoutes. func NewInternalAPI( - base *base.BaseDendrite, + processContext *process.ProcessContext, + cfg *config.Dendrite, + natsInstance *jetstream.NATSInstance, userAPI userapi.AppserviceUserAPI, rsAPI roomserverAPI.RoomserverInternalAPI, ) appserviceAPI.AppServiceInternalAPI { @@ -46,7 +49,7 @@ func NewInternalAPI( Transport: &http.Transport{ DisableKeepAlives: true, TLSClientConfig: &tls.Config{ - InsecureSkipVerify: base.Cfg.AppServiceAPI.DisableTLSValidation, + InsecureSkipVerify: cfg.AppServiceAPI.DisableTLSValidation, }, Proxy: http.ProxyFromEnvironment, }, @@ -55,21 +58,21 @@ func NewInternalAPI( // outbound and inbound requests (inbound only for the internal API) appserviceQueryAPI := &query.AppServiceQueryAPI{ HTTPClient: client, - Cfg: &base.Cfg.AppServiceAPI, + Cfg: &cfg.AppServiceAPI, ProtocolCache: map[string]appserviceAPI.ASProtocolResponse{}, CacheMu: sync.Mutex{}, } - if len(base.Cfg.Derived.ApplicationServices) == 0 { + if len(cfg.Derived.ApplicationServices) == 0 { return appserviceQueryAPI } // Wrap application services in a type that relates the application service and // a sync.Cond object that can be used to notify workers when there are new // events to be sent out. - for _, appservice := range base.Cfg.Derived.ApplicationServices { + for _, appservice := range cfg.Derived.ApplicationServices { // Create bot account for this AS if it doesn't already exist - if err := generateAppServiceAccount(userAPI, appservice, base.Cfg.Global.ServerName); err != nil { + if err := generateAppServiceAccount(userAPI, appservice, cfg.Global.ServerName); err != nil { logrus.WithFields(logrus.Fields{ "appservice": appservice.ID, }).WithError(err).Panicf("failed to generate bot account for appservice") @@ -78,9 +81,9 @@ func NewInternalAPI( // Only consume if we actually have ASes to track, else we'll just chew cycles needlessly. // We can't add ASes at runtime so this is safe to do. - js, _ := base.NATS.Prepare(base.ProcessContext, &base.Cfg.Global.JetStream) + js, _ := natsInstance.Prepare(processContext, &cfg.Global.JetStream) consumer := consumers.NewOutputRoomEventConsumer( - base.ProcessContext, &base.Cfg.AppServiceAPI, + processContext, &cfg.AppServiceAPI, client, js, rsAPI, ) if err := consumer.Start(); err != nil { diff --git a/appservice/appservice_test.go b/appservice/appservice_test.go index ad6f1dfc9c..6c8a07b5c6 100644 --- a/appservice/appservice_test.go +++ b/appservice/appservice_test.go @@ -9,12 +9,15 @@ import ( "regexp" "strings" "testing" + "time" "github.com/matrix-org/dendrite/appservice" "github.com/matrix-org/dendrite/appservice/api" "github.com/matrix-org/dendrite/internal/caching" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver" "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/dendrite/setup/jetstream" "github.com/matrix-org/dendrite/test" "github.com/matrix-org/dendrite/userapi" @@ -105,11 +108,11 @@ func TestAppserviceInternalAPI(t *testing.T) { } test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { - base, closeBase := testrig.CreateBaseDendrite(t, dbType) - defer closeBase() + cfg, ctx, close := testrig.CreateConfig(t, dbType) + defer close() // Create a dummy application service - base.Cfg.AppServiceAPI.Derived.ApplicationServices = []config.ApplicationService{ + cfg.AppServiceAPI.Derived.ApplicationServices = []config.ApplicationService{ { ID: "someID", URL: srv.URL, @@ -124,11 +127,17 @@ func TestAppserviceInternalAPI(t *testing.T) { }, } - caches := caching.NewRistrettoCache(base.Cfg.Global.Cache.EstimatedMaxSize, base.Cfg.Global.Cache.MaxAge, caching.DisableMetrics) + t.Cleanup(func() { + ctx.ShutdownDendrite() + ctx.WaitForShutdown() + }) + caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) // Create required internal APIs - rsAPI := roomserver.NewInternalAPI(base, caches) - usrAPI := userapi.NewInternalAPI(base, rsAPI, nil) - asAPI := appservice.NewInternalAPI(base, usrAPI, rsAPI) + natsInstance := jetstream.NATSInstance{} + cm := sqlutil.NewConnectionManager(ctx, cfg.Global.DatabaseOptions) + rsAPI := roomserver.NewInternalAPI(ctx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) + usrAPI := userapi.NewInternalAPI(ctx, cfg, cm, &natsInstance, rsAPI, nil) + asAPI := appservice.NewInternalAPI(ctx, cfg, &natsInstance, usrAPI, rsAPI) runCases(t, asAPI) }) diff --git a/build/dendritejs-pinecone/main.go b/build/dendritejs-pinecone/main.go index 96f034bdf4..bc9535fc1a 100644 --- a/build/dendritejs-pinecone/main.go +++ b/build/dendritejs-pinecone/main.go @@ -31,10 +31,12 @@ import ( "github.com/matrix-org/dendrite/federationapi" "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/internal/httputil" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver" "github.com/matrix-org/dendrite/setup" - "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/dendrite/setup/jetstream" + "github.com/matrix-org/dendrite/setup/process" "github.com/matrix-org/dendrite/userapi" "github.com/matrix-org/gomatrixserverlib" @@ -158,9 +160,8 @@ func startup() { pManager.AddPeer("wss://pinecone.matrix.org/public") cfg := &config.Dendrite{} - cfg.Defaults(true) + cfg.Defaults(config.DefaultOpts{Generate: true, SingleDatabase: false}) cfg.UserAPI.AccountDatabase.ConnectionString = "file:/idb/dendritejs_account.db" - cfg.AppServiceAPI.Database.ConnectionString = "file:/idb/dendritejs_appservice.db" cfg.FederationAPI.Database.ConnectionString = "file:/idb/dendritejs_fedsender.db" cfg.MediaAPI.Database.ConnectionString = "file:/idb/dendritejs_mediaapi.db" cfg.RoomServer.Database.ConnectionString = "file:/idb/dendritejs_roomserver.db" @@ -177,29 +178,30 @@ func startup() { if err := cfg.Derive(); err != nil { logrus.Fatalf("Failed to derive values from config: %s", err) } - base := base.NewBaseDendrite(cfg) - defer base.Close() // nolint: errcheck + natsInstance := jetstream.NATSInstance{} + processCtx := process.NewProcessContext() + cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) + routers := httputil.NewRouters() + caches := caching.NewRistrettoCache(cfg.Global.Cache.EstimatedMaxSize, cfg.Global.Cache.MaxAge, caching.EnableMetrics) + rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.EnableMetrics) - rsAPI := roomserver.NewInternalAPI(base) - - federation := conn.CreateFederationClient(base, pSessions) + federation := conn.CreateFederationClient(cfg, pSessions) serverKeyAPI := &signing.YggdrasilKeys{} keyRing := serverKeyAPI.KeyRing() - userAPI := userapi.NewInternalAPI(base, rsAPI, federation) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federation) asQuery := appservice.NewInternalAPI( - base, userAPI, rsAPI, + processCtx, cfg, &natsInstance, userAPI, rsAPI, ) rsAPI.SetAppserviceAPI(asQuery) - caches := caching.NewRistrettoCache(base.Cfg.Global.Cache.EstimatedMaxSize, base.Cfg.Global.Cache.MaxAge, caching.EnableMetrics) - fedSenderAPI := federationapi.NewInternalAPI(base, federation, rsAPI, caches, keyRing, true) + fedSenderAPI := federationapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, federation, rsAPI, caches, keyRing, true) rsAPI.SetFederationAPI(fedSenderAPI, keyRing) monolith := setup.Monolith{ - Config: base.Cfg, - Client: conn.CreateClient(base, pSessions), + Config: cfg, + Client: conn.CreateClient(pSessions), FedClient: federation, KeyRing: keyRing, @@ -210,15 +212,15 @@ func startup() { //ServerKeyAPI: serverKeyAPI, ExtPublicRoomsProvider: rooms.NewPineconeRoomProvider(pRouter, pSessions, fedSenderAPI, federation), } - monolith.AddAllPublicRoutes(base, caches) + monolith.AddAllPublicRoutes(processCtx, cfg, routers, cm, &natsInstance, caches, caching.EnableMetrics) httpRouter := mux.NewRouter().SkipClean(true).UseEncodedPath() - httpRouter.PathPrefix(httputil.PublicClientPathPrefix).Handler(base.Routers.Client) - httpRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.Routers.Media) + httpRouter.PathPrefix(httputil.PublicClientPathPrefix).Handler(routers.Client) + httpRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(routers.Media) p2pRouter := pSessions.Protocol("matrix").HTTP().Mux() - p2pRouter.Handle(httputil.PublicFederationPathPrefix, base.Routers.Federation) - p2pRouter.Handle(httputil.PublicMediaPathPrefix, base.Routers.Media) + p2pRouter.Handle(httputil.PublicFederationPathPrefix, routers.Federation) + p2pRouter.Handle(httputil.PublicMediaPathPrefix, routers.Media) // Expose the matrix APIs via fetch - for local traffic go func() { diff --git a/build/gobind-pinecone/monolith.go b/build/gobind-pinecone/monolith.go index 16797eec0c..2e2ca04db2 100644 --- a/build/gobind-pinecone/monolith.go +++ b/build/gobind-pinecone/monolith.go @@ -30,6 +30,9 @@ import ( "github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/relay" "github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing" "github.com/matrix-org/dendrite/federationapi/api" + "github.com/matrix-org/dendrite/internal/httputil" + "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/setup/process" userapiAPI "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/pinecone/types" @@ -187,7 +190,7 @@ func (m *DendriteMonolith) SetRelayServers(nodeID string, uris string) { relay.UpdateNodeRelayServers( gomatrixserverlib.ServerName(nodeKey), relays, - m.p2pMonolith.BaseDendrite.Context(), + m.p2pMonolith.ProcessCtx.Context(), m.p2pMonolith.GetFederationAPI(), ) } @@ -214,7 +217,7 @@ func (m *DendriteMonolith) GetRelayServers(nodeID string) string { } else { request := api.P2PQueryRelayServersRequest{Server: gomatrixserverlib.ServerName(nodeKey)} response := api.P2PQueryRelayServersResponse{} - err := m.p2pMonolith.GetFederationAPI().P2PQueryRelayServers(m.p2pMonolith.BaseDendrite.Context(), &request, &response) + err := m.p2pMonolith.GetFederationAPI().P2PQueryRelayServers(m.p2pMonolith.ProcessCtx.Context(), &request, &response) if err != nil { logrus.Warnf("Failed obtaining list of this node's relay servers: %s", err.Error()) return "" @@ -346,10 +349,14 @@ func (m *DendriteMonolith) Start() { // This isn't actually fixed: https://github.com/blevesearch/zapx/pull/147 cfg.SyncAPI.Fulltext.Enabled = false + processCtx := process.NewProcessContext() + cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) + routers := httputil.NewRouters() + enableRelaying := false enableMetrics := false enableWebsockets := false - m.p2pMonolith.SetupDendrite(cfg, 65432, enableRelaying, enableMetrics, enableWebsockets) + m.p2pMonolith.SetupDendrite(processCtx, cfg, cm, routers, 65432, enableRelaying, enableMetrics, enableWebsockets) m.p2pMonolith.StartMonolith() } diff --git a/build/gobind-yggdrasil/monolith.go b/build/gobind-yggdrasil/monolith.go index 8faad1d022..7ce1892c91 100644 --- a/build/gobind-yggdrasil/monolith.go +++ b/build/gobind-yggdrasil/monolith.go @@ -12,6 +12,7 @@ import ( "path/filepath" "time" + "github.com/getsentry/sentry-go" "github.com/gorilla/mux" "github.com/matrix-org/dendrite/appservice" "github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing" @@ -19,12 +20,15 @@ import ( "github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/yggrooms" "github.com/matrix-org/dendrite/federationapi" "github.com/matrix-org/dendrite/federationapi/api" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/internal/httputil" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver" "github.com/matrix-org/dendrite/setup" - "github.com/matrix-org/dendrite/setup/base" + basepkg "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/dendrite/setup/jetstream" "github.com/matrix-org/dendrite/setup/process" "github.com/matrix-org/dendrite/test" "github.com/matrix-org/dendrite/userapi" @@ -149,26 +153,71 @@ func (m *DendriteMonolith) Start() { panic(err) } - base := base.NewBaseDendrite(cfg) - base.ConfigureAdminEndpoints() - m.processContext = base.ProcessContext - defer base.Close() // nolint: errcheck + configErrors := &config.ConfigErrors{} + cfg.Verify(configErrors) + if len(*configErrors) > 0 { + for _, err := range *configErrors { + logrus.Errorf("Configuration error: %s", err) + } + logrus.Fatalf("Failed to start due to configuration errors") + } + + internal.SetupStdLogging() + internal.SetupHookLogging(cfg.Logging) + internal.SetupPprof() + + logrus.Infof("Dendrite version %s", internal.VersionString()) + + if !cfg.ClientAPI.RegistrationDisabled && cfg.ClientAPI.OpenRegistrationWithoutVerificationEnabled { + logrus.Warn("Open registration is enabled") + } - federation := ygg.CreateFederationClient(base) + closer, err := cfg.SetupTracing() + if err != nil { + logrus.WithError(err).Panicf("failed to start opentracing") + } + defer closer.Close() + + if cfg.Global.Sentry.Enabled { + logrus.Info("Setting up Sentry for debugging...") + err = sentry.Init(sentry.ClientOptions{ + Dsn: cfg.Global.Sentry.DSN, + Environment: cfg.Global.Sentry.Environment, + Debug: true, + ServerName: string(cfg.Global.ServerName), + Release: "dendrite@" + internal.VersionString(), + AttachStacktrace: true, + }) + if err != nil { + logrus.WithError(err).Panic("failed to start Sentry") + } + } + processCtx := process.NewProcessContext() + cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) + routers := httputil.NewRouters() + basepkg.ConfigureAdminEndpoints(processCtx, routers) + m.processContext = processCtx + defer func() { + processCtx.ShutdownDendrite() + processCtx.WaitForShutdown() + }() // nolint: errcheck + + federation := ygg.CreateFederationClient(cfg) serverKeyAPI := &signing.YggdrasilKeys{} keyRing := serverKeyAPI.KeyRing() - caches := caching.NewRistrettoCache(cfg.Global.Cache.EstimatedMaxSize, cfg.Global.Cache.MaxAge, caching.DisableMetrics) - rsAPI := roomserver.NewInternalAPI(base, caches) + caches := caching.NewRistrettoCache(cfg.Global.Cache.EstimatedMaxSize, cfg.Global.Cache.MaxAge, caching.EnableMetrics) + natsInstance := jetstream.NATSInstance{} + rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.EnableMetrics) fsAPI := federationapi.NewInternalAPI( - base, federation, rsAPI, caches, keyRing, true, + processCtx, cfg, cm, &natsInstance, federation, rsAPI, caches, keyRing, true, ) - userAPI := userapi.NewInternalAPI(base, rsAPI, federation) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federation) - asAPI := appservice.NewInternalAPI(base, userAPI, rsAPI) + asAPI := appservice.NewInternalAPI(processCtx, cfg, &natsInstance, userAPI, rsAPI) rsAPI.SetAppserviceAPI(asAPI) // The underlying roomserver implementation needs to be able to call the fedsender. @@ -176,8 +225,8 @@ func (m *DendriteMonolith) Start() { rsAPI.SetFederationAPI(fsAPI, keyRing) monolith := setup.Monolith{ - Config: base.Cfg, - Client: ygg.CreateClient(base), + Config: cfg, + Client: ygg.CreateClient(), FedClient: federation, KeyRing: keyRing, @@ -189,17 +238,17 @@ func (m *DendriteMonolith) Start() { ygg, fsAPI, federation, ), } - monolith.AddAllPublicRoutes(base, caches) + monolith.AddAllPublicRoutes(processCtx, cfg, routers, cm, &natsInstance, caches, caching.EnableMetrics) httpRouter := mux.NewRouter() - httpRouter.PathPrefix(httputil.PublicClientPathPrefix).Handler(base.Routers.Client) - httpRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.Routers.Media) - httpRouter.PathPrefix(httputil.DendriteAdminPathPrefix).Handler(base.Routers.DendriteAdmin) - httpRouter.PathPrefix(httputil.SynapseAdminPathPrefix).Handler(base.Routers.SynapseAdmin) + httpRouter.PathPrefix(httputil.PublicClientPathPrefix).Handler(routers.Client) + httpRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(routers.Media) + httpRouter.PathPrefix(httputil.DendriteAdminPathPrefix).Handler(routers.DendriteAdmin) + httpRouter.PathPrefix(httputil.SynapseAdminPathPrefix).Handler(routers.SynapseAdmin) yggRouter := mux.NewRouter() - yggRouter.PathPrefix(httputil.PublicFederationPathPrefix).Handler(base.Routers.Federation) - yggRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.Routers.Media) + yggRouter.PathPrefix(httputil.PublicFederationPathPrefix).Handler(routers.Federation) + yggRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(routers.Media) // Build both ends of a HTTP multiplex. m.httpServer = &http.Server{ diff --git a/clientapi/admin_test.go b/clientapi/admin_test.go index 46e2d30377..4d2bf67b2d 100644 --- a/clientapi/admin_test.go +++ b/clientapi/admin_test.go @@ -5,13 +5,17 @@ import ( "net/http" "net/http/httptest" "testing" + "time" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/federationapi" "github.com/matrix-org/dendrite/internal/caching" + "github.com/matrix-org/dendrite/internal/httputil" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/dendrite/setup/jetstream" "github.com/matrix-org/dendrite/syncapi" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" @@ -30,20 +34,22 @@ func TestAdminResetPassword(t *testing.T) { ctx := context.Background() test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { - base, baseClose := testrig.CreateBaseDendrite(t, dbType) - defer baseClose() - + cfg, processCtx, close := testrig.CreateConfig(t, dbType) + defer close() + natsInstance := jetstream.NATSInstance{} // add a vhost - base.Cfg.Global.VirtualHosts = append(base.Cfg.Global.VirtualHosts, &config.VirtualHost{ + cfg.Global.VirtualHosts = append(cfg.Global.VirtualHosts, &config.VirtualHost{ SigningIdentity: gomatrixserverlib.SigningIdentity{ServerName: "vh1"}, }) - caches := caching.NewRistrettoCache(base.Cfg.Global.Cache.EstimatedMaxSize, base.Cfg.Global.Cache.MaxAge, caching.DisableMetrics) - rsAPI := roomserver.NewInternalAPI(base, caches) + routers := httputil.NewRouters() + cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) + caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) + rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) // Needed for changing the password/login - userAPI := userapi.NewInternalAPI(base, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) // We mostly need the userAPI for this test, so nil for other APIs/caches etc. - AddPublicRoutes(base, nil, rsAPI, nil, nil, nil, userAPI, nil, nil) + AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) // Create the users in the userapi and login accessTokens := map[*test.User]string{ @@ -73,7 +79,7 @@ func TestAdminResetPassword(t *testing.T) { "password": password, })) rec := httptest.NewRecorder() - base.Routers.Client.ServeHTTP(rec, req) + routers.Client.ServeHTTP(rec, req) if rec.Code != http.StatusOK { t.Fatalf("failed to login: %s", rec.Body.String()) } @@ -126,7 +132,7 @@ func TestAdminResetPassword(t *testing.T) { } rec := httptest.NewRecorder() - base.Routers.DendriteAdmin.ServeHTTP(rec, req) + routers.DendriteAdmin.ServeHTTP(rec, req) t.Logf("%s", rec.Body.String()) if tc.wantOK && rec.Code != http.StatusOK { t.Fatalf("expected http status %d, got %d: %s", http.StatusOK, rec.Code, rec.Body.String()) @@ -149,17 +155,19 @@ func TestPurgeRoom(t *testing.T) { ctx := context.Background() test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { - base, baseClose := testrig.CreateBaseDendrite(t, dbType) - caches := caching.NewRistrettoCache(base.Cfg.Global.Cache.EstimatedMaxSize, base.Cfg.Global.Cache.MaxAge, caching.DisableMetrics) - defer baseClose() + cfg, processCtx, close := testrig.CreateConfig(t, dbType) + caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) + natsInstance := jetstream.NATSInstance{} + defer close() - fedClient := base.CreateFederationClient() - rsAPI := roomserver.NewInternalAPI(base, caches) - userAPI := userapi.NewInternalAPI(base, rsAPI, nil) + routers := httputil.NewRouters() + cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) + rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) // this starts the JetStream consumers - syncapi.AddPublicRoutes(base, userAPI, rsAPI, caches) - federationapi.NewInternalAPI(base, fedClient, rsAPI, caches, nil, true) + syncapi.AddPublicRoutes(processCtx, routers, cfg, cm, &natsInstance, userAPI, rsAPI, caches, caching.DisableMetrics) + federationapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, rsAPI, caches, nil, true) rsAPI.SetFederationAPI(nil, nil) // Create the room @@ -168,7 +176,7 @@ func TestPurgeRoom(t *testing.T) { } // We mostly need the rsAPI for this test, so nil for other APIs/caches etc. - AddPublicRoutes(base, nil, rsAPI, nil, nil, nil, userAPI, nil, nil) + AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) // Create the users in the userapi and login accessTokens := map[*test.User]string{ @@ -196,7 +204,7 @@ func TestPurgeRoom(t *testing.T) { "password": password, })) rec := httptest.NewRecorder() - base.Routers.Client.ServeHTTP(rec, req) + routers.Client.ServeHTTP(rec, req) if rec.Code != http.StatusOK { t.Fatalf("failed to login: %s", rec.Body.String()) } @@ -221,7 +229,7 @@ func TestPurgeRoom(t *testing.T) { req.Header.Set("Authorization", "Bearer "+accessTokens[aliceAdmin]) rec := httptest.NewRecorder() - base.Routers.DendriteAdmin.ServeHTTP(rec, req) + routers.DendriteAdmin.ServeHTTP(rec, req) t.Logf("%s", rec.Body.String()) if tc.wantOK && rec.Code != http.StatusOK { t.Fatalf("expected http status %d, got %d: %s", http.StatusOK, rec.Code, rec.Body.String()) diff --git a/clientapi/clientapi.go b/clientapi/clientapi.go index e9985d43ff..d35f1d4d14 100644 --- a/clientapi/clientapi.go +++ b/clientapi/clientapi.go @@ -15,6 +15,9 @@ package clientapi import ( + "github.com/matrix-org/dendrite/internal/httputil" + "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/dendrite/setup/process" userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" @@ -25,13 +28,15 @@ import ( federationAPI "github.com/matrix-org/dendrite/federationapi/api" "github.com/matrix-org/dendrite/internal/transactions" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" - "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/jetstream" ) // AddPublicRoutes sets up and registers HTTP handlers for the ClientAPI component. func AddPublicRoutes( - base *base.BaseDendrite, + processContext *process.ProcessContext, + routers httputil.Routers, + cfg *config.Dendrite, + natsInstance *jetstream.NATSInstance, federation *gomatrixserverlib.FederationClient, rsAPI roomserverAPI.ClientRoomserverAPI, asAPI appserviceAPI.AppServiceInternalAPI, @@ -39,27 +44,25 @@ func AddPublicRoutes( fsAPI federationAPI.ClientFederationAPI, userAPI userapi.ClientUserAPI, userDirectoryProvider userapi.QuerySearchProfilesAPI, - extRoomsProvider api.ExtraPublicRoomsProvider, + extRoomsProvider api.ExtraPublicRoomsProvider, enableMetrics bool, ) { - cfg := &base.Cfg.ClientAPI - mscCfg := &base.Cfg.MSCs - js, natsClient := base.NATS.Prepare(base.ProcessContext, &cfg.Matrix.JetStream) + js, natsClient := natsInstance.Prepare(processContext, &cfg.Global.JetStream) syncProducer := &producers.SyncAPIProducer{ JetStream: js, - TopicReceiptEvent: cfg.Matrix.JetStream.Prefixed(jetstream.OutputReceiptEvent), - TopicSendToDeviceEvent: cfg.Matrix.JetStream.Prefixed(jetstream.OutputSendToDeviceEvent), - TopicTypingEvent: cfg.Matrix.JetStream.Prefixed(jetstream.OutputTypingEvent), - TopicPresenceEvent: cfg.Matrix.JetStream.Prefixed(jetstream.OutputPresenceEvent), + TopicReceiptEvent: cfg.Global.JetStream.Prefixed(jetstream.OutputReceiptEvent), + TopicSendToDeviceEvent: cfg.Global.JetStream.Prefixed(jetstream.OutputSendToDeviceEvent), + TopicTypingEvent: cfg.Global.JetStream.Prefixed(jetstream.OutputTypingEvent), + TopicPresenceEvent: cfg.Global.JetStream.Prefixed(jetstream.OutputPresenceEvent), UserAPI: userAPI, - ServerName: cfg.Matrix.ServerName, + ServerName: cfg.Global.ServerName, } routing.Setup( - base, + routers, cfg, rsAPI, asAPI, userAPI, userDirectoryProvider, federation, syncProducer, transactionsCache, fsAPI, - extRoomsProvider, mscCfg, natsClient, + extRoomsProvider, natsClient, enableMetrics, ) } diff --git a/clientapi/routing/auth_fallback_test.go b/clientapi/routing/auth_fallback_test.go index 534581bdd8..afeca051be 100644 --- a/clientapi/routing/auth_fallback_test.go +++ b/clientapi/routing/auth_fallback_test.go @@ -10,30 +10,28 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/setup/config" - "github.com/matrix-org/dendrite/test/testrig" ) func Test_AuthFallback(t *testing.T) { - base, _, _ := testrig.Base(nil) - defer base.Close() - + cfg := config.Dendrite{} + cfg.Defaults(config.DefaultOpts{Generate: true, SingleDatabase: true}) for _, useHCaptcha := range []bool{false, true} { for _, recaptchaEnabled := range []bool{false, true} { for _, wantErr := range []bool{false, true} { t.Run(fmt.Sprintf("useHCaptcha(%v) - recaptchaEnabled(%v) - wantErr(%v)", useHCaptcha, recaptchaEnabled, wantErr), func(t *testing.T) { // Set the defaults for each test - base.Cfg.ClientAPI.Defaults(config.DefaultOpts{Generate: true, SingleDatabase: true}) - base.Cfg.ClientAPI.RecaptchaEnabled = recaptchaEnabled - base.Cfg.ClientAPI.RecaptchaPublicKey = "pub" - base.Cfg.ClientAPI.RecaptchaPrivateKey = "priv" + cfg.ClientAPI.Defaults(config.DefaultOpts{Generate: true, SingleDatabase: true}) + cfg.ClientAPI.RecaptchaEnabled = recaptchaEnabled + cfg.ClientAPI.RecaptchaPublicKey = "pub" + cfg.ClientAPI.RecaptchaPrivateKey = "priv" if useHCaptcha { - base.Cfg.ClientAPI.RecaptchaSiteVerifyAPI = "https://hcaptcha.com/siteverify" - base.Cfg.ClientAPI.RecaptchaApiJsUrl = "https://js.hcaptcha.com/1/api.js" - base.Cfg.ClientAPI.RecaptchaFormField = "h-captcha-response" - base.Cfg.ClientAPI.RecaptchaSitekeyClass = "h-captcha" + cfg.ClientAPI.RecaptchaSiteVerifyAPI = "https://hcaptcha.com/siteverify" + cfg.ClientAPI.RecaptchaApiJsUrl = "https://js.hcaptcha.com/1/api.js" + cfg.ClientAPI.RecaptchaFormField = "h-captcha-response" + cfg.ClientAPI.RecaptchaSitekeyClass = "h-captcha" } cfgErrs := &config.ConfigErrors{} - base.Cfg.ClientAPI.Verify(cfgErrs) + cfg.ClientAPI.Verify(cfgErrs) if len(*cfgErrs) > 0 { t.Fatalf("(hCaptcha=%v) unexpected config errors: %s", useHCaptcha, cfgErrs.Error()) } @@ -41,7 +39,7 @@ func Test_AuthFallback(t *testing.T) { req := httptest.NewRequest(http.MethodGet, "/?session=1337", nil) rec := httptest.NewRecorder() - AuthFallback(rec, req, authtypes.LoginTypeRecaptcha, &base.Cfg.ClientAPI) + AuthFallback(rec, req, authtypes.LoginTypeRecaptcha, &cfg.ClientAPI) if !recaptchaEnabled { if rec.Code != http.StatusBadRequest { t.Fatalf("unexpected response code: %d, want %d", rec.Code, http.StatusBadRequest) @@ -50,8 +48,8 @@ func Test_AuthFallback(t *testing.T) { t.Fatalf("unexpected response body: %s", rec.Body.String()) } } else { - if !strings.Contains(rec.Body.String(), base.Cfg.ClientAPI.RecaptchaSitekeyClass) { - t.Fatalf("body does not contain %s: %s", base.Cfg.ClientAPI.RecaptchaSitekeyClass, rec.Body.String()) + if !strings.Contains(rec.Body.String(), cfg.ClientAPI.RecaptchaSitekeyClass) { + t.Fatalf("body does not contain %s: %s", cfg.ClientAPI.RecaptchaSitekeyClass, rec.Body.String()) } } @@ -64,14 +62,14 @@ func Test_AuthFallback(t *testing.T) { })) defer srv.Close() // nolint: errcheck - base.Cfg.ClientAPI.RecaptchaSiteVerifyAPI = srv.URL + cfg.ClientAPI.RecaptchaSiteVerifyAPI = srv.URL // check the result after sending the captcha req = httptest.NewRequest(http.MethodPost, "/?session=1337", nil) req.Form = url.Values{} - req.Form.Add(base.Cfg.ClientAPI.RecaptchaFormField, "someRandomValue") + req.Form.Add(cfg.ClientAPI.RecaptchaFormField, "someRandomValue") rec = httptest.NewRecorder() - AuthFallback(rec, req, authtypes.LoginTypeRecaptcha, &base.Cfg.ClientAPI) + AuthFallback(rec, req, authtypes.LoginTypeRecaptcha, &cfg.ClientAPI) if recaptchaEnabled { if !wantErr { if rec.Code != http.StatusOK { @@ -105,7 +103,7 @@ func Test_AuthFallback(t *testing.T) { t.Run("unknown fallbacks are handled correctly", func(t *testing.T) { req := httptest.NewRequest(http.MethodPost, "/?session=1337", nil) rec := httptest.NewRecorder() - AuthFallback(rec, req, "DoesNotExist", &base.Cfg.ClientAPI) + AuthFallback(rec, req, "DoesNotExist", &cfg.ClientAPI) if rec.Code != http.StatusNotImplemented { t.Fatalf("unexpected http status: %d, want %d", rec.Code, http.StatusNotImplemented) } @@ -114,7 +112,7 @@ func Test_AuthFallback(t *testing.T) { t.Run("unknown methods are handled correctly", func(t *testing.T) { req := httptest.NewRequest(http.MethodDelete, "/?session=1337", nil) rec := httptest.NewRecorder() - AuthFallback(rec, req, authtypes.LoginTypeRecaptcha, &base.Cfg.ClientAPI) + AuthFallback(rec, req, authtypes.LoginTypeRecaptcha, &cfg.ClientAPI) if rec.Code != http.StatusMethodNotAllowed { t.Fatalf("unexpected http status: %d, want %d", rec.Code, http.StatusMethodNotAllowed) } @@ -123,7 +121,7 @@ func Test_AuthFallback(t *testing.T) { t.Run("missing session parameter is handled correctly", func(t *testing.T) { req := httptest.NewRequest(http.MethodGet, "/", nil) rec := httptest.NewRecorder() - AuthFallback(rec, req, authtypes.LoginTypeRecaptcha, &base.Cfg.ClientAPI) + AuthFallback(rec, req, authtypes.LoginTypeRecaptcha, &cfg.ClientAPI) if rec.Code != http.StatusBadRequest { t.Fatalf("unexpected http status: %d, want %d", rec.Code, http.StatusBadRequest) } @@ -132,7 +130,7 @@ func Test_AuthFallback(t *testing.T) { t.Run("missing session parameter is handled correctly", func(t *testing.T) { req := httptest.NewRequest(http.MethodGet, "/", nil) rec := httptest.NewRecorder() - AuthFallback(rec, req, authtypes.LoginTypeRecaptcha, &base.Cfg.ClientAPI) + AuthFallback(rec, req, authtypes.LoginTypeRecaptcha, &cfg.ClientAPI) if rec.Code != http.StatusBadRequest { t.Fatalf("unexpected http status: %d, want %d", rec.Code, http.StatusBadRequest) } @@ -141,7 +139,7 @@ func Test_AuthFallback(t *testing.T) { t.Run("missing 'response' is handled correctly", func(t *testing.T) { req := httptest.NewRequest(http.MethodPost, "/?session=1337", nil) rec := httptest.NewRecorder() - AuthFallback(rec, req, authtypes.LoginTypeRecaptcha, &base.Cfg.ClientAPI) + AuthFallback(rec, req, authtypes.LoginTypeRecaptcha, &cfg.ClientAPI) if rec.Code != http.StatusBadRequest { t.Fatalf("unexpected http status: %d, want %d", rec.Code, http.StatusBadRequest) } diff --git a/clientapi/routing/joinroom_test.go b/clientapi/routing/joinroom_test.go index de8f9538db..fd58ff5d53 100644 --- a/clientapi/routing/joinroom_test.go +++ b/clientapi/routing/joinroom_test.go @@ -8,6 +8,8 @@ import ( "time" "github.com/matrix-org/dendrite/internal/caching" + "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/setup/jetstream" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/dendrite/appservice" @@ -25,13 +27,15 @@ func TestJoinRoomByIDOrAlias(t *testing.T) { ctx := context.Background() test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { - base, baseClose := testrig.CreateBaseDendrite(t, dbType) - defer baseClose() + cfg, processCtx, close := testrig.CreateConfig(t, dbType) + defer close() - caches := caching.NewRistrettoCache(base.Cfg.Global.Cache.EstimatedMaxSize, base.Cfg.Global.Cache.MaxAge, caching.DisableMetrics) - rsAPI := roomserver.NewInternalAPI(base, caches) - userAPI := userapi.NewInternalAPI(base, rsAPI, nil) - asAPI := appservice.NewInternalAPI(base, userAPI, rsAPI) + cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) + caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) + natsInstance := jetstream.NATSInstance{} + rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + asAPI := appservice.NewInternalAPI(processCtx, cfg, &natsInstance, userAPI, rsAPI) rsAPI.SetFederationAPI(nil, nil) // creates the rs.Inputer etc // Create the users in the userapi @@ -63,7 +67,7 @@ func TestJoinRoomByIDOrAlias(t *testing.T) { RoomAliasName: "alias", Invite: []string{bob.ID}, GuestCanJoin: false, - }, aliceDev, &base.Cfg.ClientAPI, userAPI, rsAPI, asAPI, time.Now()) + }, aliceDev, &cfg.ClientAPI, userAPI, rsAPI, asAPI, time.Now()) crResp, ok := resp.JSON.(createRoomResponse) if !ok { t.Fatalf("response is not a createRoomResponse: %+v", resp) @@ -78,7 +82,7 @@ func TestJoinRoomByIDOrAlias(t *testing.T) { Preset: presetPublicChat, Invite: []string{charlie.ID}, GuestCanJoin: true, - }, aliceDev, &base.Cfg.ClientAPI, userAPI, rsAPI, asAPI, time.Now()) + }, aliceDev, &cfg.ClientAPI, userAPI, rsAPI, asAPI, time.Now()) crRespWithGuestAccess, ok := resp.JSON.(createRoomResponse) if !ok { t.Fatalf("response is not a createRoomResponse: %+v", resp) diff --git a/clientapi/routing/login_test.go b/clientapi/routing/login_test.go index fd3d8cba97..b27730767e 100644 --- a/clientapi/routing/login_test.go +++ b/clientapi/routing/login_test.go @@ -7,11 +7,15 @@ import ( "net/http/httptest" "strings" "testing" + "time" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/internal/caching" + "github.com/matrix-org/dendrite/internal/httputil" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver" "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/dendrite/setup/jetstream" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" @@ -29,21 +33,24 @@ func TestLogin(t *testing.T) { ctx := context.Background() test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { - base, baseClose := testrig.CreateBaseDendrite(t, dbType) - defer baseClose() - base.Cfg.ClientAPI.RateLimiting.Enabled = false + cfg, processCtx, close := testrig.CreateConfig(t, dbType) + defer close() + cfg.ClientAPI.RateLimiting.Enabled = false + natsInstance := jetstream.NATSInstance{} // add a vhost - base.Cfg.Global.VirtualHosts = append(base.Cfg.Global.VirtualHosts, &config.VirtualHost{ + cfg.Global.VirtualHosts = append(cfg.Global.VirtualHosts, &config.VirtualHost{ SigningIdentity: gomatrixserverlib.SigningIdentity{ServerName: "vh1"}, }) - caches := caching.NewRistrettoCache(base.Cfg.Global.Cache.EstimatedMaxSize, base.Cfg.Global.Cache.MaxAge, caching.DisableMetrics) - rsAPI := roomserver.NewInternalAPI(base, caches) + cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) + routers := httputil.NewRouters() + caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) + rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) // Needed for /login - userAPI := userapi.NewInternalAPI(base, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) // We mostly need the userAPI for this test, so nil for other APIs/caches etc. - Setup(base, &base.Cfg.ClientAPI, nil, nil, userAPI, nil, nil, nil, nil, nil, nil, &base.Cfg.MSCs, nil) + Setup(routers, cfg, nil, nil, userAPI, nil, nil, nil, nil, nil, nil, nil, caching.DisableMetrics) // Create password password := util.RandomString(8) @@ -116,7 +123,7 @@ func TestLogin(t *testing.T) { "password": password, })) rec := httptest.NewRecorder() - base.Routers.Client.ServeHTTP(rec, req) + routers.Client.ServeHTTP(rec, req) if tc.wantOK && rec.Code != http.StatusOK { t.Fatalf("failed to login: %s", rec.Body.String()) } diff --git a/clientapi/routing/register_test.go b/clientapi/routing/register_test.go index c06b0ae122..46cd8b2b90 100644 --- a/clientapi/routing/register_test.go +++ b/clientapi/routing/register_test.go @@ -31,8 +31,10 @@ import ( "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/caching" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver" "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/dendrite/setup/jetstream" "github.com/matrix-org/dendrite/test" "github.com/matrix-org/dendrite/test/testrig" "github.com/matrix-org/dendrite/userapi" @@ -405,12 +407,15 @@ func Test_register(t *testing.T) { } test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { - base, baseClose := testrig.CreateBaseDendrite(t, dbType) - defer baseClose() + cfg, processCtx, close := testrig.CreateConfig(t, dbType) + defer close() - caches := caching.NewRistrettoCache(base.Cfg.Global.Cache.EstimatedMaxSize, base.Cfg.Global.Cache.MaxAge, caching.DisableMetrics) - rsAPI := roomserver.NewInternalAPI(base, caches) - userAPI := userapi.NewInternalAPI(base, rsAPI, nil) + caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) + natsInstance := jetstream.NATSInstance{} + + cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) + rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { @@ -432,16 +437,16 @@ func Test_register(t *testing.T) { } })) defer srv.Close() - base.Cfg.ClientAPI.RecaptchaSiteVerifyAPI = srv.URL + cfg.ClientAPI.RecaptchaSiteVerifyAPI = srv.URL } - if err := base.Cfg.Derive(); err != nil { + if err := cfg.Derive(); err != nil { t.Fatalf("failed to derive config: %s", err) } - base.Cfg.ClientAPI.RecaptchaEnabled = tc.enableRecaptcha - base.Cfg.ClientAPI.RegistrationDisabled = tc.registrationDisabled - base.Cfg.ClientAPI.GuestsDisabled = tc.guestsDisabled + cfg.ClientAPI.RecaptchaEnabled = tc.enableRecaptcha + cfg.ClientAPI.RegistrationDisabled = tc.registrationDisabled + cfg.ClientAPI.GuestsDisabled = tc.guestsDisabled if tc.kind == "" { tc.kind = "user" @@ -469,15 +474,15 @@ func Test_register(t *testing.T) { req := httptest.NewRequest(http.MethodPost, fmt.Sprintf("/?kind=%s", tc.kind), body) - resp := Register(req, userAPI, &base.Cfg.ClientAPI) + resp := Register(req, userAPI, &cfg.ClientAPI) t.Logf("Resp: %+v", resp) // The first request should return a userInteractiveResponse switch r := resp.JSON.(type) { case userInteractiveResponse: // Check that the flows are the ones we configured - if !reflect.DeepEqual(r.Flows, base.Cfg.Derived.Registration.Flows) { - t.Fatalf("unexpected registration flows: %+v, want %+v", r.Flows, base.Cfg.Derived.Registration.Flows) + if !reflect.DeepEqual(r.Flows, cfg.Derived.Registration.Flows) { + t.Fatalf("unexpected registration flows: %+v, want %+v", r.Flows, cfg.Derived.Registration.Flows) } case *jsonerror.MatrixError: if !reflect.DeepEqual(tc.wantResponse, resp) { @@ -533,7 +538,7 @@ func Test_register(t *testing.T) { req = httptest.NewRequest(http.MethodPost, "/", body) - resp = Register(req, userAPI, &base.Cfg.ClientAPI) + resp = Register(req, userAPI, &cfg.ClientAPI) switch resp.JSON.(type) { case *jsonerror.MatrixError: @@ -576,17 +581,19 @@ func Test_register(t *testing.T) { func TestRegisterUserWithDisplayName(t *testing.T) { test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { - base, baseClose := testrig.CreateBaseDendrite(t, dbType) - defer baseClose() - base.Cfg.Global.ServerName = "server" - - caches := caching.NewRistrettoCache(base.Cfg.Global.Cache.EstimatedMaxSize, base.Cfg.Global.Cache.MaxAge, caching.DisableMetrics) - rsAPI := roomserver.NewInternalAPI(base, caches) - userAPI := userapi.NewInternalAPI(base, rsAPI, nil) + cfg, processCtx, close := testrig.CreateConfig(t, dbType) + defer close() + cfg.Global.ServerName = "server" + + caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) + natsInstance := jetstream.NATSInstance{} + cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) + rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) deviceName, deviceID := "deviceName", "deviceID" expectedDisplayName := "DisplayName" response := completeRegistration( - base.Context(), + processCtx.Context(), userAPI, "user", "server", @@ -606,7 +613,7 @@ func TestRegisterUserWithDisplayName(t *testing.T) { req := api.QueryProfileRequest{UserID: "@user:server"} var res api.QueryProfileResponse - err := userAPI.QueryProfile(base.Context(), &req, &res) + err := userAPI.QueryProfile(processCtx.Context(), &req, &res) assert.NoError(t, err) assert.Equal(t, expectedDisplayName, res.DisplayName) }) @@ -614,14 +621,17 @@ func TestRegisterUserWithDisplayName(t *testing.T) { func TestRegisterAdminUsingSharedSecret(t *testing.T) { test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { - base, baseClose := testrig.CreateBaseDendrite(t, dbType) - defer baseClose() - base.Cfg.Global.ServerName = "server" + cfg, processCtx, close := testrig.CreateConfig(t, dbType) + defer close() + natsInstance := jetstream.NATSInstance{} + cfg.Global.ServerName = "server" sharedSecret := "dendritetest" - base.Cfg.ClientAPI.RegistrationSharedSecret = sharedSecret - caches := caching.NewRistrettoCache(base.Cfg.Global.Cache.EstimatedMaxSize, base.Cfg.Global.Cache.MaxAge, caching.DisableMetrics) - rsAPI := roomserver.NewInternalAPI(base, caches) - userAPI := userapi.NewInternalAPI(base, rsAPI, nil) + cfg.ClientAPI.RegistrationSharedSecret = sharedSecret + + cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) + caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) + rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) expectedDisplayName := "rabbit" jsonStr := []byte(`{"admin":true,"mac":"24dca3bba410e43fe64b9b5c28306693bf3baa9f","nonce":"759f047f312b99ff428b21d581256f8592b8976e58bc1b543972dc6147e529a79657605b52d7becd160ff5137f3de11975684319187e06901955f79e5a6c5a79","password":"wonderland","username":"alice","displayname":"rabbit"}`) @@ -645,7 +655,7 @@ func TestRegisterAdminUsingSharedSecret(t *testing.T) { ssrr := httptest.NewRequest(http.MethodPost, "/", body) response := handleSharedSecretRegistration( - &base.Cfg.ClientAPI, + &cfg.ClientAPI, userAPI, r, ssrr, @@ -654,7 +664,7 @@ func TestRegisterAdminUsingSharedSecret(t *testing.T) { profilReq := api.QueryProfileRequest{UserID: "@alice:server"} var profileRes api.QueryProfileResponse - err = userAPI.QueryProfile(base.Context(), &profilReq, &profileRes) + err = userAPI.QueryProfile(processCtx.Context(), &profilReq, &profileRes) assert.NoError(t, err) assert.Equal(t, expectedDisplayName, profileRes.DisplayName) }) diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index 4ef2ac92e9..6a86980da8 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -21,7 +21,6 @@ import ( "sync" "github.com/gorilla/mux" - "github.com/matrix-org/dendrite/setup/base" userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" @@ -50,8 +49,8 @@ import ( // applied: // nolint: gocyclo func Setup( - base *base.BaseDendrite, - cfg *config.ClientAPI, + routers httputil.Routers, + dendriteCfg *config.Dendrite, rsAPI roomserverAPI.ClientRoomserverAPI, asAPI appserviceAPI.AppServiceInternalAPI, userAPI userapi.ClientUserAPI, @@ -61,14 +60,16 @@ func Setup( transactionsCache *transactions.Cache, federationSender federationAPI.ClientFederationAPI, extRoomsProvider api.ExtraPublicRoomsProvider, - mscCfg *config.MSCs, natsClient *nats.Conn, + natsClient *nats.Conn, enableMetrics bool, ) { - publicAPIMux := base.Routers.Client - wkMux := base.Routers.WellKnown - synapseAdminRouter := base.Routers.SynapseAdmin - dendriteAdminRouter := base.Routers.DendriteAdmin - - if base.EnableMetrics { + cfg := &dendriteCfg.ClientAPI + mscCfg := &dendriteCfg.MSCs + publicAPIMux := routers.Client + wkMux := routers.WellKnown + synapseAdminRouter := routers.SynapseAdmin + dendriteAdminRouter := routers.DendriteAdmin + + if enableMetrics { prometheus.MustRegister(amtRegUsers, sendEventDuration) } @@ -656,7 +657,7 @@ func Setup( ).Methods(http.MethodGet, http.MethodPost, http.MethodOptions) v3mux.Handle("/auth/{authType}/fallback/web", - httputil.MakeHTMLAPI("auth_fallback", base.EnableMetrics, func(w http.ResponseWriter, req *http.Request) { + httputil.MakeHTMLAPI("auth_fallback", enableMetrics, func(w http.ResponseWriter, req *http.Request) { vars := mux.Vars(req) AuthFallback(w, req, vars["authType"], cfg) }), diff --git a/cmd/dendrite-demo-pinecone/conn/client.go b/cmd/dendrite-demo-pinecone/conn/client.go index a91434f624..885de00572 100644 --- a/cmd/dendrite-demo-pinecone/conn/client.go +++ b/cmd/dendrite-demo-pinecone/conn/client.go @@ -21,7 +21,7 @@ import ( "net/http" "strings" - "github.com/matrix-org/dendrite/setup/base" + "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/gomatrixserverlib" "nhooyr.io/websocket" @@ -90,7 +90,7 @@ func createTransport(s *pineconeSessions.Sessions) *http.Transport { } func CreateClient( - base *base.BaseDendrite, s *pineconeSessions.Sessions, + s *pineconeSessions.Sessions, ) *gomatrixserverlib.Client { return gomatrixserverlib.NewClient( gomatrixserverlib.WithTransport(createTransport(s)), @@ -98,10 +98,10 @@ func CreateClient( } func CreateFederationClient( - base *base.BaseDendrite, s *pineconeSessions.Sessions, + cfg *config.Dendrite, s *pineconeSessions.Sessions, ) *gomatrixserverlib.FederationClient { return gomatrixserverlib.NewFederationClient( - base.Cfg.Global.SigningIdentities(), + cfg.Global.SigningIdentities(), gomatrixserverlib.WithTransport(createTransport(s)), ) } diff --git a/cmd/dendrite-demo-pinecone/main.go b/cmd/dendrite-demo-pinecone/main.go index 7706a73f51..7c710cbbb7 100644 --- a/cmd/dendrite-demo-pinecone/main.go +++ b/cmd/dendrite-demo-pinecone/main.go @@ -27,8 +27,11 @@ import ( "github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/monolith" "github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/httputil" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/setup" "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/dendrite/setup/process" "github.com/matrix-org/gomatrixserverlib" "github.com/sirupsen/logrus" @@ -87,9 +90,13 @@ func main() { } } + processCtx := process.NewProcessContext() + cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) + routers := httputil.NewRouters() + enableMetrics := true enableWebsockets := true - p2pMonolith.SetupDendrite(cfg, *instancePort, *instanceRelayingEnabled, enableMetrics, enableWebsockets) + p2pMonolith.SetupDendrite(processCtx, cfg, cm, routers, *instancePort, *instanceRelayingEnabled, enableMetrics, enableWebsockets) p2pMonolith.StartMonolith() p2pMonolith.WaitForShutdown() diff --git a/cmd/dendrite-demo-pinecone/monolith/monolith.go b/cmd/dendrite-demo-pinecone/monolith/monolith.go index 5781d6571e..10a3493e12 100644 --- a/cmd/dendrite-demo-pinecone/monolith/monolith.go +++ b/cmd/dendrite-demo-pinecone/monolith/monolith.go @@ -23,6 +23,7 @@ import ( "net" "net/http" "path/filepath" + "sync" "time" "github.com/gorilla/mux" @@ -39,6 +40,7 @@ import ( "github.com/matrix-org/dendrite/federationapi/producers" "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/internal/httputil" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/relayapi" relayAPI "github.com/matrix-org/dendrite/relayapi/api" "github.com/matrix-org/dendrite/roomserver" @@ -46,6 +48,7 @@ import ( "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/jetstream" + "github.com/matrix-org/dendrite/setup/process" "github.com/matrix-org/dendrite/userapi" userAPI "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" @@ -61,13 +64,13 @@ import ( const SessionProtocol = "matrix" type P2PMonolith struct { - BaseDendrite *base.BaseDendrite Sessions *pineconeSessions.Sessions Multicast *pineconeMulticast.Multicast ConnManager *pineconeConnections.ConnectionManager Router *pineconeRouter.Router EventChannel chan pineconeEvents.Event RelayRetriever relay.RelayServerRetriever + ProcessCtx *process.ProcessContext dendrite setup.Monolith port int @@ -77,6 +80,7 @@ type P2PMonolith struct { listener net.Listener httpListenAddr string stopHandlingEvents chan bool + httpServerMu sync.Mutex } func GenerateDefaultConfig(sk ed25519.PrivateKey, storageDir string, cacheDir string, dbPrefix string) *config.Dendrite { @@ -121,53 +125,52 @@ func (p *P2PMonolith) SetupPinecone(sk ed25519.PrivateKey) { p.ConnManager = pineconeConnections.NewConnectionManager(p.Router, nil) } -func (p *P2PMonolith) SetupDendrite(cfg *config.Dendrite, port int, enableRelaying bool, enableMetrics bool, enableWebsockets bool) { - if enableMetrics { - p.BaseDendrite = base.NewBaseDendrite(cfg) - } else { - p.BaseDendrite = base.NewBaseDendrite(cfg, base.DisableMetrics) - } +func (p *P2PMonolith) SetupDendrite( + processCtx *process.ProcessContext, cfg *config.Dendrite, cm sqlutil.Connections, routers httputil.Routers, + port int, enableRelaying bool, enableMetrics bool, enableWebsockets bool) { + p.port = port - p.BaseDendrite.ConfigureAdminEndpoints() + base.ConfigureAdminEndpoints(processCtx, routers) - federation := conn.CreateFederationClient(p.BaseDendrite, p.Sessions) + federation := conn.CreateFederationClient(cfg, p.Sessions) serverKeyAPI := &signing.YggdrasilKeys{} keyRing := serverKeyAPI.KeyRing() caches := caching.NewRistrettoCache(cfg.Global.Cache.EstimatedMaxSize, cfg.Global.Cache.MaxAge, enableMetrics) - rsAPI := roomserver.NewInternalAPI(p.BaseDendrite, caches) + natsInstance := jetstream.NATSInstance{} + rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, enableMetrics) fsAPI := federationapi.NewInternalAPI( - p.BaseDendrite, federation, rsAPI, caches, keyRing, true, + processCtx, cfg, cm, &natsInstance, federation, rsAPI, caches, keyRing, true, ) - userAPI := userapi.NewInternalAPI(p.BaseDendrite, rsAPI, federation) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federation) - asAPI := appservice.NewInternalAPI(p.BaseDendrite, userAPI, rsAPI) + asAPI := appservice.NewInternalAPI(processCtx, cfg, &natsInstance, userAPI, rsAPI) rsAPI.SetFederationAPI(fsAPI, keyRing) userProvider := users.NewPineconeUserProvider(p.Router, p.Sessions, userAPI, federation) roomProvider := rooms.NewPineconeRoomProvider(p.Router, p.Sessions, fsAPI, federation) - js, _ := p.BaseDendrite.NATS.Prepare(p.BaseDendrite.ProcessContext, &p.BaseDendrite.Cfg.Global.JetStream) + js, _ := natsInstance.Prepare(processCtx, &cfg.Global.JetStream) producer := &producers.SyncAPIProducer{ JetStream: js, - TopicReceiptEvent: p.BaseDendrite.Cfg.Global.JetStream.Prefixed(jetstream.OutputReceiptEvent), - TopicSendToDeviceEvent: p.BaseDendrite.Cfg.Global.JetStream.Prefixed(jetstream.OutputSendToDeviceEvent), - TopicTypingEvent: p.BaseDendrite.Cfg.Global.JetStream.Prefixed(jetstream.OutputTypingEvent), - TopicPresenceEvent: p.BaseDendrite.Cfg.Global.JetStream.Prefixed(jetstream.OutputPresenceEvent), - TopicDeviceListUpdate: p.BaseDendrite.Cfg.Global.JetStream.Prefixed(jetstream.InputDeviceListUpdate), - TopicSigningKeyUpdate: p.BaseDendrite.Cfg.Global.JetStream.Prefixed(jetstream.InputSigningKeyUpdate), - Config: &p.BaseDendrite.Cfg.FederationAPI, + TopicReceiptEvent: cfg.Global.JetStream.Prefixed(jetstream.OutputReceiptEvent), + TopicSendToDeviceEvent: cfg.Global.JetStream.Prefixed(jetstream.OutputSendToDeviceEvent), + TopicTypingEvent: cfg.Global.JetStream.Prefixed(jetstream.OutputTypingEvent), + TopicPresenceEvent: cfg.Global.JetStream.Prefixed(jetstream.OutputPresenceEvent), + TopicDeviceListUpdate: cfg.Global.JetStream.Prefixed(jetstream.InputDeviceListUpdate), + TopicSigningKeyUpdate: cfg.Global.JetStream.Prefixed(jetstream.InputSigningKeyUpdate), + Config: &cfg.FederationAPI, UserAPI: userAPI, } - relayAPI := relayapi.NewRelayInternalAPI(p.BaseDendrite, federation, rsAPI, keyRing, producer, enableRelaying, caches) + relayAPI := relayapi.NewRelayInternalAPI(cfg, cm, federation, rsAPI, keyRing, producer, enableRelaying, caches) logrus.Infof("Relaying enabled: %v", relayAPI.RelayingEnabled()) p.dendrite = setup.Monolith{ - Config: p.BaseDendrite.Cfg, - Client: conn.CreateClient(p.BaseDendrite, p.Sessions), + Config: cfg, + Client: conn.CreateClient(p.Sessions), FedClient: federation, KeyRing: keyRing, @@ -179,9 +182,10 @@ func (p *P2PMonolith) SetupDendrite(cfg *config.Dendrite, port int, enableRelayi ExtPublicRoomsProvider: roomProvider, ExtUserDirectoryProvider: userProvider, } - p.dendrite.AddAllPublicRoutes(p.BaseDendrite, caches) + p.ProcessCtx = processCtx + p.dendrite.AddAllPublicRoutes(processCtx, cfg, routers, cm, &natsInstance, caches, enableMetrics) - p.setupHttpServers(userProvider, enableWebsockets) + p.setupHttpServers(userProvider, routers, enableWebsockets) } func (p *P2PMonolith) GetFederationAPI() federationAPI.FederationInternalAPI { @@ -203,20 +207,22 @@ func (p *P2PMonolith) StartMonolith() { func (p *P2PMonolith) Stop() { logrus.Info("Stopping monolith") - _ = p.BaseDendrite.Close() + p.ProcessCtx.ShutdownDendrite() p.WaitForShutdown() logrus.Info("Stopped monolith") } func (p *P2PMonolith) WaitForShutdown() { - p.BaseDendrite.WaitForShutdown() + p.ProcessCtx.WaitForShutdown() p.closeAllResources() } func (p *P2PMonolith) closeAllResources() { logrus.Info("Closing monolith resources") + p.httpServerMu.Lock() if p.httpServer != nil { _ = p.httpServer.Shutdown(context.Background()) + p.httpServerMu.Unlock() } select { @@ -246,12 +252,12 @@ func (p *P2PMonolith) Addr() string { return p.httpListenAddr } -func (p *P2PMonolith) setupHttpServers(userProvider *users.PineconeUserProvider, enableWebsockets bool) { +func (p *P2PMonolith) setupHttpServers(userProvider *users.PineconeUserProvider, routers httputil.Routers, enableWebsockets bool) { p.httpMux = mux.NewRouter().SkipClean(true).UseEncodedPath() - p.httpMux.PathPrefix(httputil.PublicClientPathPrefix).Handler(p.BaseDendrite.Routers.Client) - p.httpMux.PathPrefix(httputil.PublicMediaPathPrefix).Handler(p.BaseDendrite.Routers.Media) - p.httpMux.PathPrefix(httputil.DendriteAdminPathPrefix).Handler(p.BaseDendrite.Routers.DendriteAdmin) - p.httpMux.PathPrefix(httputil.SynapseAdminPathPrefix).Handler(p.BaseDendrite.Routers.SynapseAdmin) + p.httpMux.PathPrefix(httputil.PublicClientPathPrefix).Handler(routers.Client) + p.httpMux.PathPrefix(httputil.PublicMediaPathPrefix).Handler(routers.Media) + p.httpMux.PathPrefix(httputil.DendriteAdminPathPrefix).Handler(routers.DendriteAdmin) + p.httpMux.PathPrefix(httputil.SynapseAdminPathPrefix).Handler(routers.SynapseAdmin) if enableWebsockets { wsUpgrader := websocket.Upgrader{ @@ -284,8 +290,8 @@ func (p *P2PMonolith) setupHttpServers(userProvider *users.PineconeUserProvider, p.pineconeMux = mux.NewRouter().SkipClean(true).UseEncodedPath() p.pineconeMux.PathPrefix(users.PublicURL).HandlerFunc(userProvider.FederatedUserProfiles) - p.pineconeMux.PathPrefix(httputil.PublicFederationPathPrefix).Handler(p.BaseDendrite.Routers.Federation) - p.pineconeMux.PathPrefix(httputil.PublicMediaPathPrefix).Handler(p.BaseDendrite.Routers.Media) + p.pineconeMux.PathPrefix(httputil.PublicFederationPathPrefix).Handler(routers.Federation) + p.pineconeMux.PathPrefix(httputil.PublicMediaPathPrefix).Handler(routers.Media) pHTTP := p.Sessions.Protocol(SessionProtocol).HTTP() pHTTP.Mux().Handle(users.PublicURL, p.pineconeMux) @@ -295,6 +301,7 @@ func (p *P2PMonolith) setupHttpServers(userProvider *users.PineconeUserProvider, func (p *P2PMonolith) startHTTPServers() { go func() { + p.httpServerMu.Lock() // Build both ends of a HTTP multiplex. p.httpServer = &http.Server{ Addr: ":0", @@ -307,7 +314,7 @@ func (p *P2PMonolith) startHTTPServers() { }, Handler: p.pineconeMux, } - + p.httpServerMu.Unlock() pubkey := p.Router.PublicKey() pubkeyString := hex.EncodeToString(pubkey[:]) logrus.Info("Listening on ", pubkeyString) @@ -369,7 +376,7 @@ func (p *P2PMonolith) startEventHandler() { ServerNames: []gomatrixserverlib.ServerName{gomatrixserverlib.ServerName(e.PeerID)}, } res := &federationAPI.PerformWakeupServersResponse{} - if err := p.dendrite.FederationAPI.PerformWakeupServers(p.BaseDendrite.Context(), req, res); err != nil { + if err := p.dendrite.FederationAPI.PerformWakeupServers(p.ProcessCtx.Context(), req, res); err != nil { eLog.WithError(err).Error("Failed to wakeup destination", e.PeerID) } } diff --git a/cmd/dendrite-demo-yggdrasil/main.go b/cmd/dendrite-demo-yggdrasil/main.go index 62184719a9..9a195990cf 100644 --- a/cmd/dendrite-demo-yggdrasil/main.go +++ b/cmd/dendrite-demo-yggdrasil/main.go @@ -27,7 +27,11 @@ import ( "path/filepath" "time" + "github.com/getsentry/sentry-go" "github.com/matrix-org/dendrite/internal/caching" + "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/setup/jetstream" + "github.com/matrix-org/dendrite/setup/process" "github.com/matrix-org/gomatrixserverlib" "github.com/gorilla/mux" @@ -42,7 +46,7 @@ import ( "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/roomserver" "github.com/matrix-org/dendrite/setup" - "github.com/matrix-org/dendrite/setup/base" + basepkg "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/mscs" "github.com/matrix-org/dendrite/test" @@ -58,6 +62,7 @@ var ( instanceDir = flag.String("dir", ".", "the directory to store the databases in (if --config not specified)") ) +// nolint: gocyclo func main() { flag.Parse() internal.SetupPprof() @@ -143,36 +148,83 @@ func main() { cfg.Global.ServerName = gomatrixserverlib.ServerName(hex.EncodeToString(pk)) cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID) - base := base.NewBaseDendrite(cfg) - base.ConfigureAdminEndpoints() - defer base.Close() // nolint: errcheck + configErrors := &config.ConfigErrors{} + cfg.Verify(configErrors) + if len(*configErrors) > 0 { + for _, err := range *configErrors { + logrus.Errorf("Configuration error: %s", err) + } + logrus.Fatalf("Failed to start due to configuration errors") + } + + internal.SetupStdLogging() + internal.SetupHookLogging(cfg.Logging) + internal.SetupPprof() + + logrus.Infof("Dendrite version %s", internal.VersionString()) + + if !cfg.ClientAPI.RegistrationDisabled && cfg.ClientAPI.OpenRegistrationWithoutVerificationEnabled { + logrus.Warn("Open registration is enabled") + } + + closer, err := cfg.SetupTracing() + if err != nil { + logrus.WithError(err).Panicf("failed to start opentracing") + } + defer closer.Close() // nolint: errcheck + + if cfg.Global.Sentry.Enabled { + logrus.Info("Setting up Sentry for debugging...") + err = sentry.Init(sentry.ClientOptions{ + Dsn: cfg.Global.Sentry.DSN, + Environment: cfg.Global.Sentry.Environment, + Debug: true, + ServerName: string(cfg.Global.ServerName), + Release: "dendrite@" + internal.VersionString(), + AttachStacktrace: true, + }) + if err != nil { + logrus.WithError(err).Panic("failed to start Sentry") + } + } + + processCtx := process.NewProcessContext() + cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) + routers := httputil.NewRouters() + + basepkg.ConfigureAdminEndpoints(processCtx, routers) + defer func() { + processCtx.ShutdownDendrite() + processCtx.WaitForShutdown() + }() // nolint: errcheck ygg, err := yggconn.Setup(sk, *instanceName, ".", *instancePeer, *instanceListen) if err != nil { panic(err) } - federation := ygg.CreateFederationClient(base) + federation := ygg.CreateFederationClient(cfg) serverKeyAPI := &signing.YggdrasilKeys{} keyRing := serverKeyAPI.KeyRing() - caches := caching.NewRistrettoCache(base.Cfg.Global.Cache.EstimatedMaxSize, base.Cfg.Global.Cache.MaxAge, caching.EnableMetrics) - rsAPI := roomserver.NewInternalAPI(base, caches) + caches := caching.NewRistrettoCache(cfg.Global.Cache.EstimatedMaxSize, cfg.Global.Cache.MaxAge, caching.EnableMetrics) + natsInstance := jetstream.NATSInstance{} + rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.EnableMetrics) - userAPI := userapi.NewInternalAPI(base, rsAPI, federation) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federation) - asAPI := appservice.NewInternalAPI(base, userAPI, rsAPI) + asAPI := appservice.NewInternalAPI(processCtx, cfg, &natsInstance, userAPI, rsAPI) rsAPI.SetAppserviceAPI(asAPI) fsAPI := federationapi.NewInternalAPI( - base, federation, rsAPI, caches, keyRing, true, + processCtx, cfg, cm, &natsInstance, federation, rsAPI, caches, keyRing, true, ) rsAPI.SetFederationAPI(fsAPI, keyRing) monolith := setup.Monolith{ - Config: base.Cfg, - Client: ygg.CreateClient(base), + Config: cfg, + Client: ygg.CreateClient(), FedClient: federation, KeyRing: keyRing, @@ -184,21 +236,21 @@ func main() { ygg, fsAPI, federation, ), } - monolith.AddAllPublicRoutes(base, caches) - if err := mscs.Enable(base, &monolith, caches); err != nil { + monolith.AddAllPublicRoutes(processCtx, cfg, routers, cm, &natsInstance, caches, caching.EnableMetrics) + if err := mscs.Enable(cfg, cm, routers, &monolith, caches); err != nil { logrus.WithError(err).Fatalf("Failed to enable MSCs") } httpRouter := mux.NewRouter().SkipClean(true).UseEncodedPath() - httpRouter.PathPrefix(httputil.PublicClientPathPrefix).Handler(base.Routers.Client) - httpRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.Routers.Media) - httpRouter.PathPrefix(httputil.DendriteAdminPathPrefix).Handler(base.Routers.DendriteAdmin) - httpRouter.PathPrefix(httputil.SynapseAdminPathPrefix).Handler(base.Routers.SynapseAdmin) + httpRouter.PathPrefix(httputil.PublicClientPathPrefix).Handler(routers.Client) + httpRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(routers.Media) + httpRouter.PathPrefix(httputil.DendriteAdminPathPrefix).Handler(routers.DendriteAdmin) + httpRouter.PathPrefix(httputil.SynapseAdminPathPrefix).Handler(routers.SynapseAdmin) embed.Embed(httpRouter, *instancePort, "Yggdrasil Demo") yggRouter := mux.NewRouter().SkipClean(true).UseEncodedPath() - yggRouter.PathPrefix(httputil.PublicFederationPathPrefix).Handler(base.Routers.Federation) - yggRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.Routers.Media) + yggRouter.PathPrefix(httputil.PublicFederationPathPrefix).Handler(routers.Federation) + yggRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(routers.Media) // Build both ends of a HTTP multiplex. httpServer := &http.Server{ @@ -232,5 +284,5 @@ func main() { }() // We want to block forever to let the HTTP and HTTPS handler serve the APIs - base.WaitForShutdown() + basepkg.WaitForShutdown(processCtx) } diff --git a/cmd/dendrite-demo-yggdrasil/yggconn/client.go b/cmd/dendrite-demo-yggdrasil/yggconn/client.go index 41a9ec1231..51365547b7 100644 --- a/cmd/dendrite-demo-yggdrasil/yggconn/client.go +++ b/cmd/dendrite-demo-yggdrasil/yggconn/client.go @@ -4,7 +4,7 @@ import ( "net/http" "time" - "github.com/matrix-org/dendrite/setup/base" + "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/gomatrixserverlib" ) @@ -17,9 +17,7 @@ func (y *yggroundtripper) RoundTrip(req *http.Request) (*http.Response, error) { return y.inner.RoundTrip(req) } -func (n *Node) CreateClient( - base *base.BaseDendrite, -) *gomatrixserverlib.Client { +func (n *Node) CreateClient() *gomatrixserverlib.Client { tr := &http.Transport{} tr.RegisterProtocol( "matrix", &yggroundtripper{ @@ -39,7 +37,7 @@ func (n *Node) CreateClient( } func (n *Node) CreateFederationClient( - base *base.BaseDendrite, + cfg *config.Dendrite, ) *gomatrixserverlib.FederationClient { tr := &http.Transport{} tr.RegisterProtocol( @@ -55,7 +53,7 @@ func (n *Node) CreateFederationClient( }, ) return gomatrixserverlib.NewFederationClient( - base.Cfg.Global.SigningIdentities(), + cfg.Global.SigningIdentities(), gomatrixserverlib.WithTransport(tr), ) } diff --git a/cmd/dendrite-upgrade-tests/main.go b/cmd/dendrite-upgrade-tests/main.go index 174a80a3e3..6a0e217990 100644 --- a/cmd/dendrite-upgrade-tests/main.go +++ b/cmd/dendrite-upgrade-tests/main.go @@ -259,10 +259,20 @@ func buildDendrite(httpClient *http.Client, dockerClient *client.Client, tmpDir func getAndSortVersionsFromGithub(httpClient *http.Client) (semVers []*semver.Version, err error) { u := "https://api.github.com/repos/matrix-org/dendrite/tags" - res, err := httpClient.Get(u) - if err != nil { - return nil, err + + var res *http.Response + for i := 0; i < 3; i++ { + res, err = httpClient.Get(u) + if err != nil { + return nil, err + } + if res.StatusCode == 200 { + break + } + log.Printf("Github API returned HTTP %d, retrying\n", res.StatusCode) + time.Sleep(time.Second * 5) } + if res.StatusCode != 200 { return nil, fmt.Errorf("%s returned HTTP %d", u, res.StatusCode) } diff --git a/cmd/dendrite/main.go b/cmd/dendrite/main.go index 29290eb985..a22677ebe0 100644 --- a/cmd/dendrite/main.go +++ b/cmd/dendrite/main.go @@ -16,8 +16,16 @@ package main import ( "flag" + "time" + "github.com/getsentry/sentry-go" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/caching" + "github.com/matrix-org/dendrite/internal/httputil" + "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/setup/jetstream" + "github.com/matrix-org/dendrite/setup/process" + "github.com/matrix-org/gomatrixserverlib" "github.com/sirupsen/logrus" "github.com/matrix-org/dendrite/appservice" @@ -66,25 +74,90 @@ func main() { httpAddr = socket } - options := []basepkg.BaseDendriteOptions{} + configErrors := &config.ConfigErrors{} + cfg.Verify(configErrors) + if len(*configErrors) > 0 { + for _, err := range *configErrors { + logrus.Errorf("Configuration error: %s", err) + } + logrus.Fatalf("Failed to start due to configuration errors") + } + processCtx := process.NewProcessContext() - base := basepkg.NewBaseDendrite(cfg, options...) - defer base.Close() // nolint: errcheck + internal.SetupStdLogging() + internal.SetupHookLogging(cfg.Logging) + internal.SetupPprof() - federation := base.CreateFederationClient() + basepkg.PlatformSanityChecks() + + logrus.Infof("Dendrite version %s", internal.VersionString()) + if !cfg.ClientAPI.RegistrationDisabled && cfg.ClientAPI.OpenRegistrationWithoutVerificationEnabled { + logrus.Warn("Open registration is enabled") + } - caches := caching.NewRistrettoCache(base.Cfg.Global.Cache.EstimatedMaxSize, base.Cfg.Global.Cache.MaxAge, caching.EnableMetrics) + // create DNS cache + var dnsCache *gomatrixserverlib.DNSCache + if cfg.Global.DNSCache.Enabled { + dnsCache = gomatrixserverlib.NewDNSCache( + cfg.Global.DNSCache.CacheSize, + cfg.Global.DNSCache.CacheLifetime, + ) + logrus.Infof( + "DNS cache enabled (size %d, lifetime %s)", + cfg.Global.DNSCache.CacheSize, + cfg.Global.DNSCache.CacheLifetime, + ) + } - rsAPI := roomserver.NewInternalAPI(base, caches) + // setup tracing + closer, err := cfg.SetupTracing() + if err != nil { + logrus.WithError(err).Panicf("failed to start opentracing") + } + defer closer.Close() // nolint: errcheck + + // setup sentry + if cfg.Global.Sentry.Enabled { + logrus.Info("Setting up Sentry for debugging...") + err = sentry.Init(sentry.ClientOptions{ + Dsn: cfg.Global.Sentry.DSN, + Environment: cfg.Global.Sentry.Environment, + Debug: true, + ServerName: string(cfg.Global.ServerName), + Release: "dendrite@" + internal.VersionString(), + AttachStacktrace: true, + }) + if err != nil { + logrus.WithError(err).Panic("failed to start Sentry") + } + go func() { + processCtx.ComponentStarted() + <-processCtx.WaitForShutdown() + if !sentry.Flush(time.Second * 5) { + logrus.Warnf("failed to flush all Sentry events!") + } + processCtx.ComponentFinished() + }() + } + + federationClient := basepkg.CreateFederationClient(cfg, dnsCache) + httpClient := basepkg.CreateClient(cfg, dnsCache) + + // prepare required dependencies + cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) + routers := httputil.NewRouters() + + caches := caching.NewRistrettoCache(cfg.Global.Cache.EstimatedMaxSize, cfg.Global.Cache.MaxAge, caching.EnableMetrics) + natsInstance := jetstream.NATSInstance{} + rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.EnableMetrics) fsAPI := federationapi.NewInternalAPI( - base, federation, rsAPI, caches, nil, false, + processCtx, cfg, cm, &natsInstance, federationClient, rsAPI, caches, nil, false, ) keyRing := fsAPI.KeyRing() - userAPI := userapi.NewInternalAPI(base, rsAPI, federation) - - asAPI := appservice.NewInternalAPI(base, userAPI, rsAPI) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federationClient) + asAPI := appservice.NewInternalAPI(processCtx, cfg, &natsInstance, userAPI, rsAPI) // The underlying roomserver implementation needs to be able to call the fedsender. // This is different to rsAPI which can be the http client which doesn't need this @@ -94,9 +167,9 @@ func main() { rsAPI.SetUserAPI(userAPI) monolith := setup.Monolith{ - Config: base.Cfg, - Client: base.CreateClient(), - FedClient: federation, + Config: cfg, + Client: httpClient, + FedClient: federationClient, KeyRing: keyRing, AppserviceAPI: asAPI, @@ -106,25 +179,25 @@ func main() { RoomserverAPI: rsAPI, UserAPI: userAPI, } - monolith.AddAllPublicRoutes(base, caches) + monolith.AddAllPublicRoutes(processCtx, cfg, routers, cm, &natsInstance, caches, caching.EnableMetrics) - if len(base.Cfg.MSCs.MSCs) > 0 { - if err := mscs.Enable(base, &monolith, caches); err != nil { + if len(cfg.MSCs.MSCs) > 0 { + if err := mscs.Enable(cfg, cm, routers, &monolith, caches); err != nil { logrus.WithError(err).Fatalf("Failed to enable MSCs") } } // Expose the matrix APIs directly rather than putting them under a /api path. go func() { - base.SetupAndServeHTTP(httpAddr, nil, nil) + basepkg.SetupAndServeHTTP(processCtx, cfg, routers, httpAddr, nil, nil) }() // Handle HTTPS if certificate and key are provided if *unixSocket == "" && *certFile != "" && *keyFile != "" { go func() { - base.SetupAndServeHTTP(httpsAddr, certFile, keyFile) + basepkg.SetupAndServeHTTP(processCtx, cfg, routers, httpsAddr, certFile, keyFile) }() } // We want to block forever to let the HTTP and HTTPS handler serve the APIs - base.WaitForShutdown() + basepkg.WaitForShutdown(processCtx) } diff --git a/cmd/resolve-state/main.go b/cmd/resolve-state/main.go index 099daaa9a3..09c0e69079 100644 --- a/cmd/resolve-state/main.go +++ b/cmd/resolve-state/main.go @@ -10,12 +10,13 @@ import ( "time" "github.com/matrix-org/dendrite/internal/caching" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver/state" "github.com/matrix-org/dendrite/roomserver/storage" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/dendrite/setup" - "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/dendrite/setup/process" "github.com/matrix-org/gomatrixserverlib" ) @@ -40,7 +41,7 @@ func main() { Level: "error", }) cfg.ClientAPI.RegistrationDisabled = true - base := base.NewBaseDendrite(cfg, base.DisableMetrics) + args := flag.Args() fmt.Println("Room version", *roomVersion) @@ -54,8 +55,10 @@ func main() { fmt.Println("Fetching", len(snapshotNIDs), "snapshot NIDs") + processCtx := process.NewProcessContext() + cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) roomserverDB, err := storage.Open( - base.ProcessContext.Context(), base.ConnectionManager, &cfg.RoomServer.Database, + processCtx.Context(), cm, &cfg.RoomServer.Database, caching.NewRistrettoCache(128*1024*1024, time.Hour, true), ) if err != nil { diff --git a/federationapi/federationapi.go b/federationapi/federationapi.go index 8a4237bae5..3b5394a136 100644 --- a/federationapi/federationapi.go +++ b/federationapi/federationapi.go @@ -17,6 +17,10 @@ package federationapi import ( "time" + "github.com/matrix-org/dendrite/internal/httputil" + "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/dendrite/setup/process" "github.com/sirupsen/logrus" "github.com/matrix-org/dendrite/federationapi/api" @@ -29,7 +33,6 @@ import ( "github.com/matrix-org/dendrite/federationapi/storage" "github.com/matrix-org/dendrite/internal/caching" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" - "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/jetstream" userapi "github.com/matrix-org/dendrite/userapi/api" @@ -40,17 +43,21 @@ import ( // AddPublicRoutes sets up and registers HTTP handlers on the base API muxes for the FederationAPI component. func AddPublicRoutes( - base *base.BaseDendrite, + processContext *process.ProcessContext, + routers httputil.Routers, + dendriteConfig *config.Dendrite, + natsInstance *jetstream.NATSInstance, userAPI userapi.FederationUserAPI, federation *gomatrixserverlib.FederationClient, keyRing gomatrixserverlib.JSONVerifier, rsAPI roomserverAPI.FederationRoomserverAPI, fedAPI federationAPI.FederationInternalAPI, servers federationAPI.ServersInRoomProvider, + enableMetrics bool, ) { - cfg := &base.Cfg.FederationAPI - mscCfg := &base.Cfg.MSCs - js, _ := base.NATS.Prepare(base.ProcessContext, &cfg.Matrix.JetStream) + cfg := &dendriteConfig.FederationAPI + mscCfg := &dendriteConfig.MSCs + js, _ := natsInstance.Prepare(processContext, &cfg.Matrix.JetStream) producer := &producers.SyncAPIProducer{ JetStream: js, TopicReceiptEvent: cfg.Matrix.JetStream.Prefixed(jetstream.OutputReceiptEvent), @@ -75,26 +82,30 @@ func AddPublicRoutes( } routing.Setup( - base, + routers, + dendriteConfig, rsAPI, f, keyRing, federation, userAPI, mscCfg, - servers, producer, + servers, producer, enableMetrics, ) } // NewInternalAPI returns a concerete implementation of the internal API. Callers // can call functions directly on the returned API or via an HTTP interface using AddInternalRoutes. func NewInternalAPI( - base *base.BaseDendrite, + processContext *process.ProcessContext, + dendriteCfg *config.Dendrite, + cm sqlutil.Connections, + natsInstance *jetstream.NATSInstance, federation api.FederationClient, rsAPI roomserverAPI.FederationRoomserverAPI, caches *caching.Caches, keyRing *gomatrixserverlib.KeyRing, resetBlacklist bool, ) api.FederationInternalAPI { - cfg := &base.Cfg.FederationAPI + cfg := &dendriteCfg.FederationAPI - federationDB, err := storage.NewDatabase(base.ProcessContext.Context(), base.ConnectionManager, &cfg.Database, caches, base.Cfg.Global.IsLocalServerName) + federationDB, err := storage.NewDatabase(processContext.Context(), cm, &cfg.Database, caches, dendriteCfg.Global.IsLocalServerName) if err != nil { logrus.WithError(err).Panic("failed to connect to federation sender db") } @@ -108,51 +119,51 @@ func NewInternalAPI( cfg.FederationMaxRetries+1, cfg.P2PFederationRetriesUntilAssumedOffline+1) - js, nats := base.NATS.Prepare(base.ProcessContext, &cfg.Matrix.JetStream) + js, nats := natsInstance.Prepare(processContext, &cfg.Matrix.JetStream) - signingInfo := base.Cfg.Global.SigningIdentities() + signingInfo := dendriteCfg.Global.SigningIdentities() queues := queue.NewOutgoingQueues( - federationDB, base.ProcessContext, + federationDB, processContext, cfg.Matrix.DisableFederation, cfg.Matrix.ServerName, federation, rsAPI, &stats, signingInfo, ) rsConsumer := consumers.NewOutputRoomEventConsumer( - base.ProcessContext, cfg, js, nats, queues, + processContext, cfg, js, nats, queues, federationDB, rsAPI, ) if err = rsConsumer.Start(); err != nil { logrus.WithError(err).Panic("failed to start room server consumer") } tsConsumer := consumers.NewOutputSendToDeviceConsumer( - base.ProcessContext, cfg, js, queues, federationDB, + processContext, cfg, js, queues, federationDB, ) if err = tsConsumer.Start(); err != nil { logrus.WithError(err).Panic("failed to start send-to-device consumer") } receiptConsumer := consumers.NewOutputReceiptConsumer( - base.ProcessContext, cfg, js, queues, federationDB, + processContext, cfg, js, queues, federationDB, ) if err = receiptConsumer.Start(); err != nil { logrus.WithError(err).Panic("failed to start receipt consumer") } typingConsumer := consumers.NewOutputTypingConsumer( - base.ProcessContext, cfg, js, queues, federationDB, + processContext, cfg, js, queues, federationDB, ) if err = typingConsumer.Start(); err != nil { logrus.WithError(err).Panic("failed to start typing consumer") } keyConsumer := consumers.NewKeyChangeConsumer( - base.ProcessContext, &base.Cfg.KeyServer, js, queues, federationDB, rsAPI, + processContext, &dendriteCfg.KeyServer, js, queues, federationDB, rsAPI, ) if err = keyConsumer.Start(); err != nil { logrus.WithError(err).Panic("failed to start key server consumer") } presenceConsumer := consumers.NewOutputPresenceConsumer( - base.ProcessContext, cfg, js, queues, federationDB, rsAPI, + processContext, cfg, js, queues, federationDB, rsAPI, ) if err = presenceConsumer.Start(); err != nil { logrus.WithError(err).Panic("failed to start presence consumer") @@ -161,7 +172,7 @@ func NewInternalAPI( var cleanExpiredEDUs func() cleanExpiredEDUs = func() { logrus.Infof("Cleaning expired EDUs") - if err := federationDB.DeleteExpiredEDUs(base.Context()); err != nil { + if err := federationDB.DeleteExpiredEDUs(processContext.Context()); err != nil { logrus.WithError(err).Error("Failed to clean expired EDUs") } time.AfterFunc(time.Hour, cleanExpiredEDUs) diff --git a/federationapi/federationapi_keys_test.go b/federationapi/federationapi_keys_test.go index bb6ee89358..425cdbb972 100644 --- a/federationapi/federationapi_keys_test.go +++ b/federationapi/federationapi_keys_test.go @@ -12,12 +12,14 @@ import ( "testing" "time" + "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/setup/jetstream" + "github.com/matrix-org/dendrite/setup/process" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/dendrite/federationapi/api" "github.com/matrix-org/dendrite/federationapi/routing" "github.com/matrix-org/dendrite/internal/caching" - "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/config" ) @@ -65,7 +67,7 @@ func TestMain(m *testing.M) { // Create a new cache but don't enable prometheus! s.cache = caching.NewRistrettoCache(8*1024*1024, time.Hour, false) - + natsInstance := jetstream.NATSInstance{} // Create a temporary directory for JetStream. d, err := os.MkdirTemp("./", "jetstream*") if err != nil { @@ -109,8 +111,9 @@ func TestMain(m *testing.M) { ) // Finally, build the server key APIs. - sbase := base.NewBaseDendrite(cfg, base.DisableMetrics) - s.api = NewInternalAPI(sbase, s.fedclient, nil, s.cache, nil, true) + processCtx := process.NewProcessContext() + cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) + s.api = NewInternalAPI(processCtx, cfg, cm, &natsInstance, s.fedclient, nil, s.cache, nil, true) } // Now that we have built our server key APIs, start the diff --git a/federationapi/federationapi_test.go b/federationapi/federationapi_test.go index 8aea96a731..d39f40512c 100644 --- a/federationapi/federationapi_test.go +++ b/federationapi/federationapi_test.go @@ -11,6 +11,8 @@ import ( "time" "github.com/matrix-org/dendrite/internal/caching" + "github.com/matrix-org/dendrite/internal/httputil" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrix" "github.com/matrix-org/gomatrixserverlib" "github.com/nats-io/nats.go" @@ -19,8 +21,6 @@ import ( "github.com/matrix-org/dendrite/federationapi/api" "github.com/matrix-org/dendrite/federationapi/internal" rsapi "github.com/matrix-org/dendrite/roomserver/api" - "github.com/matrix-org/dendrite/setup/base" - "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/jetstream" "github.com/matrix-org/dendrite/test" "github.com/matrix-org/dendrite/test/testrig" @@ -163,22 +163,24 @@ func TestFederationAPIJoinThenKeyUpdate(t *testing.T) { } func testFederationAPIJoinThenKeyUpdate(t *testing.T, dbType test.DBType) { - base, close := testrig.CreateBaseDendrite(t, dbType) - caches := caching.NewRistrettoCache(base.Cfg.Global.Cache.EstimatedMaxSize, base.Cfg.Global.Cache.MaxAge, caching.DisableMetrics) - base.Cfg.FederationAPI.PreferDirectFetch = true - base.Cfg.FederationAPI.KeyPerspectives = nil + cfg, processCtx, close := testrig.CreateConfig(t, dbType) + cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) + caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) + natsInstance := jetstream.NATSInstance{} + cfg.FederationAPI.PreferDirectFetch = true + cfg.FederationAPI.KeyPerspectives = nil defer close() - jsctx, _ := base.NATS.Prepare(base.ProcessContext, &base.Cfg.Global.JetStream) - defer jetstream.DeleteAllStreams(jsctx, &base.Cfg.Global.JetStream) + jsctx, _ := natsInstance.Prepare(processCtx, &cfg.Global.JetStream) + defer jetstream.DeleteAllStreams(jsctx, &cfg.Global.JetStream) serverA := gomatrixserverlib.ServerName("server.a") serverAKeyID := gomatrixserverlib.KeyID("ed25519:servera") serverAPrivKey := test.PrivateKeyA creator := test.NewUser(t, test.WithSigningServer(serverA, serverAKeyID, serverAPrivKey)) - myServer := base.Cfg.Global.ServerName - myServerKeyID := base.Cfg.Global.KeyID - myServerPrivKey := base.Cfg.Global.PrivateKey + myServer := cfg.Global.ServerName + myServerKeyID := cfg.Global.KeyID + myServerPrivKey := cfg.Global.PrivateKey joiningUser := test.NewUser(t, test.WithSigningServer(myServer, myServerKeyID, myServerPrivKey)) fmt.Printf("creator: %v joining user: %v\n", creator.ID, joiningUser.ID) room := test.NewRoom(t, creator) @@ -214,7 +216,7 @@ func testFederationAPIJoinThenKeyUpdate(t *testing.T, dbType test.DBType) { }, }, } - fsapi := federationapi.NewInternalAPI(base, fc, rsapi, caches, nil, false) + fsapi := federationapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, fc, rsapi, caches, nil, false) var resp api.PerformJoinResponse fsapi.PerformJoin(context.Background(), &api.PerformJoinRequest{ @@ -247,7 +249,7 @@ func testFederationAPIJoinThenKeyUpdate(t *testing.T, dbType test.DBType) { } msg := &nats.Msg{ - Subject: base.Cfg.Global.JetStream.Prefixed(jetstream.OutputKeyChangeEvent), + Subject: cfg.Global.JetStream.Prefixed(jetstream.OutputKeyChangeEvent), Header: nats.Header{}, Data: b, } @@ -265,30 +267,6 @@ func testFederationAPIJoinThenKeyUpdate(t *testing.T, dbType test.DBType) { // Tests that event IDs with '/' in them (escaped as %2F) are correctly passed to the right handler and don't 404. // Relevant for v3 rooms and a cause of flakey sytests as the IDs are randomly generated. func TestRoomsV3URLEscapeDoNot404(t *testing.T) { - _, privKey, _ := ed25519.GenerateKey(nil) - cfg := &config.Dendrite{} - cfg.Defaults(config.DefaultOpts{ - Generate: true, - SingleDatabase: false, - }) - cfg.Global.KeyID = gomatrixserverlib.KeyID("ed25519:auto") - cfg.Global.ServerName = gomatrixserverlib.ServerName("localhost") - cfg.Global.PrivateKey = privKey - cfg.Global.JetStream.InMemory = true - b := base.NewBaseDendrite(cfg, base.DisableMetrics) - keyRing := &test.NopJSONVerifier{} - // TODO: This is pretty fragile, as if anything calls anything on these nils this test will break. - // Unfortunately, it makes little sense to instantiate these dependencies when we just want to test routing. - federationapi.AddPublicRoutes(b, nil, nil, keyRing, nil, &internal.FederationInternalAPI{}, nil) - baseURL, cancel := test.ListenAndServe(t, b.Routers.Federation, true) - defer cancel() - serverName := gomatrixserverlib.ServerName(strings.TrimPrefix(baseURL, "https://")) - - fedCli := gomatrixserverlib.NewFederationClient( - cfg.Global.SigningIdentities(), - gomatrixserverlib.WithSkipVerify(true), - ) - testCases := []struct { roomVer gomatrixserverlib.RoomVersion eventJSON string @@ -317,6 +295,29 @@ func TestRoomsV3URLEscapeDoNot404(t *testing.T) { }, } + cfg, processCtx, close := testrig.CreateConfig(t, test.DBTypeSQLite) + defer close() + routers := httputil.NewRouters() + + _, privKey, _ := ed25519.GenerateKey(nil) + cfg.Global.KeyID = gomatrixserverlib.KeyID("ed25519:auto") + cfg.Global.ServerName = gomatrixserverlib.ServerName("localhost") + cfg.Global.PrivateKey = privKey + cfg.Global.JetStream.InMemory = true + keyRing := &test.NopJSONVerifier{} + natsInstance := jetstream.NATSInstance{} + // TODO: This is pretty fragile, as if anything calls anything on these nils this test will break. + // Unfortunately, it makes little sense to instantiate these dependencies when we just want to test routing. + federationapi.AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, nil, keyRing, nil, &internal.FederationInternalAPI{}, nil, caching.DisableMetrics) + baseURL, cancel := test.ListenAndServe(t, routers.Federation, true) + defer cancel() + serverName := gomatrixserverlib.ServerName(strings.TrimPrefix(baseURL, "https://")) + + fedCli := gomatrixserverlib.NewFederationClient( + cfg.Global.SigningIdentities(), + gomatrixserverlib.WithSkipVerify(true), + ) + for _, tc := range testCases { ev, err := gomatrixserverlib.NewEventFromTrustedJSON([]byte(tc.eventJSON), false, tc.roomVer) if err != nil { diff --git a/federationapi/queue/queue_test.go b/federationapi/queue/queue_test.go index c3fb10561c..55d1df4a24 100644 --- a/federationapi/queue/queue_test.go +++ b/federationapi/queue/queue_test.go @@ -22,6 +22,7 @@ import ( "time" "github.com/matrix-org/dendrite/internal/caching" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/test/testrig" "go.uber.org/atomic" "gotest.tools/v3/poll" @@ -41,18 +42,19 @@ import ( func mustCreateFederationDatabase(t *testing.T, dbType test.DBType, realDatabase bool) (storage.Database, *process.ProcessContext, func()) { if realDatabase { // Real Database/s - b, baseClose := testrig.CreateBaseDendrite(t, dbType) - caches := caching.NewRistrettoCache(b.Cfg.Global.Cache.EstimatedMaxSize, b.Cfg.Global.Cache.MaxAge, caching.DisableMetrics) + cfg, processCtx, close := testrig.CreateConfig(t, dbType) + cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) + caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) connStr, dbClose := test.PrepareDBConnectionString(t, dbType) - db, err := storage.NewDatabase(b.ProcessContext.Context(), b.ConnectionManager, &config.DatabaseOptions{ + db, err := storage.NewDatabase(processCtx.Context(), cm, &config.DatabaseOptions{ ConnectionString: config.DataSource(connStr), - }, caches, b.Cfg.Global.IsLocalServerName) + }, caches, cfg.Global.IsLocalServerName) if err != nil { t.Fatalf("NewDatabase returned %s", err) } - return db, b.ProcessContext, func() { + return db, processCtx, func() { + close() dbClose() - baseClose() } } else { // Fake Database diff --git a/federationapi/routing/profile_test.go b/federationapi/routing/profile_test.go index df494a7432..d5e9997faf 100644 --- a/federationapi/routing/profile_test.go +++ b/federationapi/routing/profile_test.go @@ -27,7 +27,10 @@ import ( fedAPI "github.com/matrix-org/dendrite/federationapi" fedInternal "github.com/matrix-org/dendrite/federationapi/internal" "github.com/matrix-org/dendrite/federationapi/routing" + "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/internal/httputil" + "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/setup/jetstream" "github.com/matrix-org/dendrite/test" "github.com/matrix-org/dendrite/test/testrig" userAPI "github.com/matrix-org/dendrite/userapi/api" @@ -46,23 +49,26 @@ func (u *fakeUserAPI) QueryProfile(ctx context.Context, req *userAPI.QueryProfil func TestHandleQueryProfile(t *testing.T) { test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { - base, close := testrig.CreateBaseDendrite(t, dbType) + cfg, processCtx, close := testrig.CreateConfig(t, dbType) + cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) + routers := httputil.NewRouters() defer close() fedMux := mux.NewRouter().SkipClean(true).PathPrefix(httputil.PublicFederationPathPrefix).Subrouter().UseEncodedPath() - base.Routers.Federation = fedMux - base.Cfg.FederationAPI.Matrix.SigningIdentity.ServerName = testOrigin - base.Cfg.FederationAPI.Matrix.Metrics.Enabled = false + natsInstance := jetstream.NATSInstance{} + routers.Federation = fedMux + cfg.FederationAPI.Matrix.SigningIdentity.ServerName = testOrigin + cfg.FederationAPI.Matrix.Metrics.Enabled = false fedClient := fakeFedClient{} serverKeyAPI := &signing.YggdrasilKeys{} keyRing := serverKeyAPI.KeyRing() - fedapi := fedAPI.NewInternalAPI(base, &fedClient, nil, nil, keyRing, true) + fedapi := fedAPI.NewInternalAPI(processCtx, cfg, cm, &natsInstance, &fedClient, nil, nil, keyRing, true) userapi := fakeUserAPI{} r, ok := fedapi.(*fedInternal.FederationInternalAPI) if !ok { panic("This is a programming error.") } - routing.Setup(base, nil, r, keyRing, &fedClient, &userapi, &base.Cfg.MSCs, nil, nil) + routing.Setup(routers, cfg, nil, r, keyRing, &fedClient, &userapi, &cfg.MSCs, nil, nil, caching.DisableMetrics) handler := fedMux.Get(routing.QueryProfileRouteName).GetHandler().ServeHTTP _, sk, _ := ed25519.GenerateKey(nil) diff --git a/federationapi/routing/query_test.go b/federationapi/routing/query_test.go index 69cf7047d6..6e702963f6 100644 --- a/federationapi/routing/query_test.go +++ b/federationapi/routing/query_test.go @@ -28,7 +28,10 @@ import ( fedclient "github.com/matrix-org/dendrite/federationapi/api" fedInternal "github.com/matrix-org/dendrite/federationapi/internal" "github.com/matrix-org/dendrite/federationapi/routing" + "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/internal/httputil" + "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/setup/jetstream" "github.com/matrix-org/dendrite/test" "github.com/matrix-org/dendrite/test/testrig" "github.com/matrix-org/gomatrixserverlib" @@ -46,23 +49,26 @@ func (f *fakeFedClient) LookupRoomAlias(ctx context.Context, origin, s gomatrixs func TestHandleQueryDirectory(t *testing.T) { test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { - base, close := testrig.CreateBaseDendrite(t, dbType) + cfg, processCtx, close := testrig.CreateConfig(t, dbType) + cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) + routers := httputil.NewRouters() defer close() fedMux := mux.NewRouter().SkipClean(true).PathPrefix(httputil.PublicFederationPathPrefix).Subrouter().UseEncodedPath() - base.Routers.Federation = fedMux - base.Cfg.FederationAPI.Matrix.SigningIdentity.ServerName = testOrigin - base.Cfg.FederationAPI.Matrix.Metrics.Enabled = false + natsInstance := jetstream.NATSInstance{} + routers.Federation = fedMux + cfg.FederationAPI.Matrix.SigningIdentity.ServerName = testOrigin + cfg.FederationAPI.Matrix.Metrics.Enabled = false fedClient := fakeFedClient{} serverKeyAPI := &signing.YggdrasilKeys{} keyRing := serverKeyAPI.KeyRing() - fedapi := fedAPI.NewInternalAPI(base, &fedClient, nil, nil, keyRing, true) + fedapi := fedAPI.NewInternalAPI(processCtx, cfg, cm, &natsInstance, &fedClient, nil, nil, keyRing, true) userapi := fakeUserAPI{} r, ok := fedapi.(*fedInternal.FederationInternalAPI) if !ok { panic("This is a programming error.") } - routing.Setup(base, nil, r, keyRing, &fedClient, &userapi, &base.Cfg.MSCs, nil, nil) + routing.Setup(routers, cfg, nil, r, keyRing, &fedClient, &userapi, &cfg.MSCs, nil, nil, caching.DisableMetrics) handler := fedMux.Get(routing.QueryDirectoryRouteName).GetHandler().ServeHTTP _, sk, _ := ed25519.GenerateKey(nil) diff --git a/federationapi/routing/routing.go b/federationapi/routing/routing.go index c86d18d2f9..a1f943e776 100644 --- a/federationapi/routing/routing.go +++ b/federationapi/routing/routing.go @@ -31,7 +31,6 @@ import ( "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/roomserver/api" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" - "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/config" userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" @@ -55,7 +54,8 @@ const ( // applied: // nolint: gocyclo func Setup( - base *base.BaseDendrite, + routers httputil.Routers, + dendriteCfg *config.Dendrite, rsAPI roomserverAPI.FederationRoomserverAPI, fsAPI *fedInternal.FederationInternalAPI, keys gomatrixserverlib.JSONVerifier, @@ -63,14 +63,14 @@ func Setup( userAPI userapi.FederationUserAPI, mscCfg *config.MSCs, servers federationAPI.ServersInRoomProvider, - producer *producers.SyncAPIProducer, + producer *producers.SyncAPIProducer, enableMetrics bool, ) { - fedMux := base.Routers.Federation - keyMux := base.Routers.Keys - wkMux := base.Routers.WellKnown - cfg := &base.Cfg.FederationAPI + fedMux := routers.Federation + keyMux := routers.Keys + wkMux := routers.WellKnown + cfg := &dendriteCfg.FederationAPI - if base.EnableMetrics { + if enableMetrics { prometheus.MustRegister( internal.PDUCountTotal, internal.EDUCountTotal, ) diff --git a/federationapi/routing/send_test.go b/federationapi/routing/send_test.go index 53e1399ab3..28fa6d6d22 100644 --- a/federationapi/routing/send_test.go +++ b/federationapi/routing/send_test.go @@ -25,7 +25,10 @@ import ( fedAPI "github.com/matrix-org/dendrite/federationapi" fedInternal "github.com/matrix-org/dendrite/federationapi/internal" "github.com/matrix-org/dendrite/federationapi/routing" + "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/internal/httputil" + "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/setup/jetstream" "github.com/matrix-org/dendrite/test" "github.com/matrix-org/dendrite/test/testrig" "github.com/matrix-org/gomatrixserverlib" @@ -44,21 +47,24 @@ type sendContent struct { func TestHandleSend(t *testing.T) { test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { - base, close := testrig.CreateBaseDendrite(t, dbType) + cfg, processCtx, close := testrig.CreateConfig(t, dbType) + cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) + routers := httputil.NewRouters() defer close() fedMux := mux.NewRouter().SkipClean(true).PathPrefix(httputil.PublicFederationPathPrefix).Subrouter().UseEncodedPath() - base.Routers.Federation = fedMux - base.Cfg.FederationAPI.Matrix.SigningIdentity.ServerName = testOrigin - base.Cfg.FederationAPI.Matrix.Metrics.Enabled = false - fedapi := fedAPI.NewInternalAPI(base, nil, nil, nil, nil, true) + natsInstance := jetstream.NATSInstance{} + routers.Federation = fedMux + cfg.FederationAPI.Matrix.SigningIdentity.ServerName = testOrigin + cfg.FederationAPI.Matrix.Metrics.Enabled = false + fedapi := fedAPI.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, nil, nil, nil, true) serverKeyAPI := &signing.YggdrasilKeys{} keyRing := serverKeyAPI.KeyRing() r, ok := fedapi.(*fedInternal.FederationInternalAPI) if !ok { panic("This is a programming error.") } - routing.Setup(base, nil, r, keyRing, nil, nil, &base.Cfg.MSCs, nil, nil) + routing.Setup(routers, cfg, nil, r, keyRing, nil, nil, &cfg.MSCs, nil, nil, caching.DisableMetrics) handler := fedMux.Get(routing.SendRouteName).GetHandler().ServeHTTP _, sk, _ := ed25519.GenerateKey(nil) diff --git a/federationapi/storage/storage_test.go b/federationapi/storage/storage_test.go index a02b71b60c..74863c07c3 100644 --- a/federationapi/storage/storage_test.go +++ b/federationapi/storage/storage_test.go @@ -20,7 +20,7 @@ func mustCreateFederationDatabase(t *testing.T, dbType test.DBType) (storage.Dat caches := caching.NewRistrettoCache(8*1024*1024, time.Hour, false) connStr, dbClose := test.PrepareDBConnectionString(t, dbType) ctx := context.Background() - cm := sqlutil.NewConnectionManager() + cm := sqlutil.NewConnectionManager(nil, config.DatabaseOptions{}) db, err := storage.NewDatabase(ctx, cm, &config.DatabaseOptions{ ConnectionString: config.DataSource(connStr), }, caches, func(server gomatrixserverlib.ServerName) bool { return server == "localhost" }) diff --git a/internal/sqlutil/connection_manager.go b/internal/sqlutil/connection_manager.go index cefd9f8081..934a2954a1 100644 --- a/internal/sqlutil/connection_manager.go +++ b/internal/sqlutil/connection_manager.go @@ -19,15 +19,21 @@ import ( "fmt" "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/dendrite/setup/process" ) type Connections struct { - db *sql.DB - writer Writer + db *sql.DB + writer Writer + globalConfig config.DatabaseOptions + processContext *process.ProcessContext } -func NewConnectionManager() Connections { - return Connections{} +func NewConnectionManager(processCtx *process.ProcessContext, globalConfig config.DatabaseOptions) Connections { + return Connections{ + globalConfig: globalConfig, + processContext: processCtx, + } } func (c *Connections) Connection(dbProperties *config.DatabaseOptions) (*sql.DB, Writer, error) { @@ -35,14 +41,29 @@ func (c *Connections) Connection(dbProperties *config.DatabaseOptions) (*sql.DB, if dbProperties.ConnectionString.IsSQLite() { writer = NewExclusiveWriter() } + var err error + if dbProperties.ConnectionString == "" { + // if no connectionString was provided, try the global one + dbProperties = &c.globalConfig + } if dbProperties.ConnectionString != "" || c.db == nil { - var err error // Open a new database connection using the supplied config. c.db, err = Open(dbProperties, writer) if err != nil { return nil, nil, err } c.writer = writer + go func() { + if c.processContext == nil { + return + } + // If we have a ProcessContext, start a component and wait for + // Dendrite to shut down to cleanly close the database connection. + c.processContext.ComponentStarted() + <-c.processContext.WaitForShutdown() + _ = c.db.Close() + c.processContext.ComponentFinished() + }() return c.db, c.writer, nil } if c.db != nil && c.writer != nil { diff --git a/internal/sqlutil/connection_manager_test.go b/internal/sqlutil/connection_manager_test.go index 610629d5ea..a9ac8d57f4 100644 --- a/internal/sqlutil/connection_manager_test.go +++ b/internal/sqlutil/connection_manager_test.go @@ -13,9 +13,9 @@ func TestConnectionManager(t *testing.T) { test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { conStr, close := test.PrepareDBConnectionString(t, dbType) t.Cleanup(close) - cm := sqlutil.NewConnectionManager() + cm := sqlutil.NewConnectionManager(nil, config.DatabaseOptions{}) - dbProps := &config.DatabaseOptions{ConnectionString: config.DataSource(string(conStr))} + dbProps := &config.DatabaseOptions{ConnectionString: config.DataSource(conStr)} db, writer, err := cm.Connection(dbProps) if err != nil { t.Fatal(err) @@ -47,8 +47,8 @@ func TestConnectionManager(t *testing.T) { } // test invalid connection string configured - cm = sqlutil.NewConnectionManager() - _, _, err = cm.Connection(&config.DatabaseOptions{ConnectionString: "http://"}) + cm2 := sqlutil.NewConnectionManager(nil, config.DatabaseOptions{}) + _, _, err = cm2.Connection(&config.DatabaseOptions{ConnectionString: "http://"}) if err == nil { t.Fatal("expected an error but got none") } diff --git a/mediaapi/mediaapi.go b/mediaapi/mediaapi.go index 42e0ea88fb..5d517ef228 100644 --- a/mediaapi/mediaapi.go +++ b/mediaapi/mediaapi.go @@ -15,9 +15,11 @@ package mediaapi import ( + "github.com/gorilla/mux" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/mediaapi/routing" "github.com/matrix-org/dendrite/mediaapi/storage" - "github.com/matrix-org/dendrite/setup/base" + "github.com/matrix-org/dendrite/setup/config" userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" "github.com/sirupsen/logrus" @@ -25,19 +27,18 @@ import ( // AddPublicRoutes sets up and registers HTTP handlers for the MediaAPI component. func AddPublicRoutes( - base *base.BaseDendrite, + mediaRouter *mux.Router, + cm sqlutil.Connections, + cfg *config.Dendrite, userAPI userapi.MediaUserAPI, client *gomatrixserverlib.Client, ) { - cfg := &base.Cfg.MediaAPI - rateCfg := &base.Cfg.ClientAPI.RateLimiting - - mediaDB, err := storage.NewMediaAPIDatasource(base.ConnectionManager, &cfg.Database) + mediaDB, err := storage.NewMediaAPIDatasource(cm, &cfg.MediaAPI.Database) if err != nil { logrus.WithError(err).Panicf("failed to connect to media db") } routing.Setup( - base.Routers.Media, cfg, rateCfg, mediaDB, userAPI, client, + mediaRouter, cfg, mediaDB, userAPI, client, ) } diff --git a/mediaapi/routing/routing.go b/mediaapi/routing/routing.go index 8a4a5afcb7..98e6a82edd 100644 --- a/mediaapi/routing/routing.go +++ b/mediaapi/routing/routing.go @@ -45,13 +45,12 @@ type configResponse struct { // nolint: gocyclo func Setup( publicAPIMux *mux.Router, - cfg *config.MediaAPI, - rateLimit *config.RateLimiting, + cfg *config.Dendrite, db storage.Database, userAPI userapi.MediaUserAPI, client *gomatrixserverlib.Client, ) { - rateLimits := httputil.NewRateLimits(rateLimit) + rateLimits := httputil.NewRateLimits(&cfg.ClientAPI.RateLimiting) v3mux := publicAPIMux.PathPrefix("/{apiversion:(?:r0|v1|v3)}/").Subrouter() @@ -65,7 +64,7 @@ func Setup( if r := rateLimits.Limit(req, dev); r != nil { return *r } - return Upload(req, cfg, dev, db, activeThumbnailGeneration) + return Upload(req, &cfg.MediaAPI, dev, db, activeThumbnailGeneration) }, ) @@ -73,8 +72,8 @@ func Setup( if r := rateLimits.Limit(req, device); r != nil { return *r } - respondSize := &cfg.MaxFileSizeBytes - if cfg.MaxFileSizeBytes == 0 { + respondSize := &cfg.MediaAPI.MaxFileSizeBytes + if cfg.MediaAPI.MaxFileSizeBytes == 0 { respondSize = nil } return util.JSONResponse{ @@ -90,12 +89,12 @@ func Setup( MXCToResult: map[string]*types.RemoteRequestResult{}, } - downloadHandler := makeDownloadAPI("download", cfg, rateLimits, db, client, activeRemoteRequests, activeThumbnailGeneration) + downloadHandler := makeDownloadAPI("download", &cfg.MediaAPI, rateLimits, db, client, activeRemoteRequests, activeThumbnailGeneration) v3mux.Handle("/download/{serverName}/{mediaId}", downloadHandler).Methods(http.MethodGet, http.MethodOptions) v3mux.Handle("/download/{serverName}/{mediaId}/{downloadName}", downloadHandler).Methods(http.MethodGet, http.MethodOptions) v3mux.Handle("/thumbnail/{serverName}/{mediaId}", - makeDownloadAPI("thumbnail", cfg, rateLimits, db, client, activeRemoteRequests, activeThumbnailGeneration), + makeDownloadAPI("thumbnail", &cfg.MediaAPI, rateLimits, db, client, activeRemoteRequests, activeThumbnailGeneration), ).Methods(http.MethodGet, http.MethodOptions) } diff --git a/mediaapi/routing/upload_test.go b/mediaapi/routing/upload_test.go index d4fb45d1b4..d088950caa 100644 --- a/mediaapi/routing/upload_test.go +++ b/mediaapi/routing/upload_test.go @@ -50,7 +50,7 @@ func Test_uploadRequest_doUpload(t *testing.T) { // create testdata folder and remove when done _ = os.Mkdir(testdataPath, os.ModePerm) defer fileutils.RemoveDir(types.Path(testdataPath), nil) - cm := sqlutil.NewConnectionManager() + cm := sqlutil.NewConnectionManager(nil, config.DatabaseOptions{}) db, err := storage.NewMediaAPIDatasource(cm, &config.DatabaseOptions{ ConnectionString: "file::memory:?cache=shared", MaxOpenConnections: 100, diff --git a/mediaapi/storage/storage_test.go b/mediaapi/storage/storage_test.go index 11febd2756..8cd29a54db 100644 --- a/mediaapi/storage/storage_test.go +++ b/mediaapi/storage/storage_test.go @@ -14,7 +14,7 @@ import ( func mustCreateDatabase(t *testing.T, dbType test.DBType) (storage.Database, func()) { connStr, close := test.PrepareDBConnectionString(t, dbType) - cm := sqlutil.NewConnectionManager() + cm := sqlutil.NewConnectionManager(nil, config.DatabaseOptions{}) db, err := storage.NewMediaAPIDatasource(cm, &config.DatabaseOptions{ ConnectionString: config.DataSource(connStr), }) diff --git a/relayapi/relayapi.go b/relayapi/relayapi.go index 925fc031da..fdc366e4c3 100644 --- a/relayapi/relayapi.go +++ b/relayapi/relayapi.go @@ -17,24 +17,25 @@ package relayapi import ( "github.com/matrix-org/dendrite/federationapi/producers" "github.com/matrix-org/dendrite/internal/caching" + "github.com/matrix-org/dendrite/internal/httputil" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/relayapi/api" "github.com/matrix-org/dendrite/relayapi/internal" "github.com/matrix-org/dendrite/relayapi/routing" "github.com/matrix-org/dendrite/relayapi/storage" rsAPI "github.com/matrix-org/dendrite/roomserver/api" - "github.com/matrix-org/dendrite/setup/base" + "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/gomatrixserverlib" "github.com/sirupsen/logrus" ) // AddPublicRoutes sets up and registers HTTP handlers on the base API muxes for the FederationAPI component. func AddPublicRoutes( - base *base.BaseDendrite, + routers httputil.Routers, + dendriteCfg *config.Dendrite, keyRing gomatrixserverlib.JSONVerifier, relayAPI api.RelayInternalAPI, ) { - fedCfg := &base.Cfg.FederationAPI - relay, ok := relayAPI.(*internal.RelayInternalAPI) if !ok { panic("relayapi.AddPublicRoutes called with a RelayInternalAPI impl which was not " + @@ -42,15 +43,16 @@ func AddPublicRoutes( } routing.Setup( - base.Routers.Federation, - fedCfg, + routers.Federation, + &dendriteCfg.FederationAPI, relay, keyRing, ) } func NewRelayInternalAPI( - base *base.BaseDendrite, + dendriteCfg *config.Dendrite, + cm sqlutil.Connections, fedClient *gomatrixserverlib.FederationClient, rsAPI rsAPI.RoomserverInternalAPI, keyRing *gomatrixserverlib.KeyRing, @@ -58,8 +60,7 @@ func NewRelayInternalAPI( relayingEnabled bool, caches caching.FederationCache, ) api.RelayInternalAPI { - cfg := &base.Cfg.RelayAPI - relayDB, err := storage.NewDatabase(base.ConnectionManager, &cfg.Database, caches, base.Cfg.Global.IsLocalServerName) + relayDB, err := storage.NewDatabase(cm, &dendriteCfg.RelayAPI.Database, caches, dendriteCfg.Global.IsLocalServerName) if err != nil { logrus.WithError(err).Panic("failed to connect to relay db") } @@ -70,8 +71,8 @@ func NewRelayInternalAPI( rsAPI, keyRing, producer, - base.Cfg.Global.Presence.EnableInbound, - base.Cfg.Global.ServerName, + dendriteCfg.Global.Presence.EnableInbound, + dendriteCfg.Global.ServerName, relayingEnabled, ) } diff --git a/relayapi/relayapi_test.go b/relayapi/relayapi_test.go index e812030988..8973c8cc94 100644 --- a/relayapi/relayapi_test.go +++ b/relayapi/relayapi_test.go @@ -21,10 +21,13 @@ import ( "net/http" "net/http/httptest" "testing" + "time" "github.com/gorilla/mux" "github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing" "github.com/matrix-org/dendrite/internal/caching" + "github.com/matrix-org/dendrite/internal/httputil" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/relayapi" "github.com/matrix-org/dendrite/test" "github.com/matrix-org/dendrite/test/testrig" @@ -34,38 +37,38 @@ import ( func TestCreateNewRelayInternalAPI(t *testing.T) { test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { - base, close := testrig.CreateBaseDendrite(t, dbType) - caches := caching.NewRistrettoCache(base.Cfg.Global.Cache.EstimatedMaxSize, base.Cfg.Global.Cache.MaxAge, caching.DisableMetrics) + cfg, processCtx, close := testrig.CreateConfig(t, dbType) + caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) defer close() - - relayAPI := relayapi.NewRelayInternalAPI(base, nil, nil, nil, nil, true, caches) + cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) + relayAPI := relayapi.NewRelayInternalAPI(cfg, cm, nil, nil, nil, nil, true, caches) assert.NotNil(t, relayAPI) }) } func TestCreateRelayInternalInvalidDatabasePanics(t *testing.T) { test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { - base, close := testrig.CreateBaseDendrite(t, dbType) + cfg, processCtx, close := testrig.CreateConfig(t, dbType) if dbType == test.DBTypeSQLite { - base.Cfg.RelayAPI.Database.ConnectionString = "file:" + cfg.RelayAPI.Database.ConnectionString = "file:" } else { - base.Cfg.RelayAPI.Database.ConnectionString = "test" + cfg.RelayAPI.Database.ConnectionString = "test" } defer close() - + cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) assert.Panics(t, func() { - relayapi.NewRelayInternalAPI(base, nil, nil, nil, nil, true, nil) + relayapi.NewRelayInternalAPI(cfg, cm, nil, nil, nil, nil, true, nil) }) }) } func TestCreateInvalidRelayPublicRoutesPanics(t *testing.T) { test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { - base, close := testrig.CreateBaseDendrite(t, dbType) + cfg, _, close := testrig.CreateConfig(t, dbType) defer close() - + routers := httputil.NewRouters() assert.Panics(t, func() { - relayapi.AddPublicRoutes(base, nil, nil) + relayapi.AddPublicRoutes(routers, cfg, nil, nil) }) }) } @@ -107,16 +110,19 @@ func createSendRelayTxnHTTPRequest(serverName gomatrixserverlib.ServerName, txnI func TestCreateRelayPublicRoutes(t *testing.T) { test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { - base, close := testrig.CreateBaseDendrite(t, dbType) + cfg, processCtx, close := testrig.CreateConfig(t, dbType) defer close() - caches := caching.NewRistrettoCache(base.Cfg.Global.Cache.EstimatedMaxSize, base.Cfg.Global.Cache.MaxAge, caching.DisableMetrics) + routers := httputil.NewRouters() + caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) - relayAPI := relayapi.NewRelayInternalAPI(base, nil, nil, nil, nil, true, caches) + cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) + + relayAPI := relayapi.NewRelayInternalAPI(cfg, cm, nil, nil, nil, nil, true, caches) assert.NotNil(t, relayAPI) serverKeyAPI := &signing.YggdrasilKeys{} keyRing := serverKeyAPI.KeyRing() - relayapi.AddPublicRoutes(base, keyRing, relayAPI) + relayapi.AddPublicRoutes(routers, cfg, keyRing, relayAPI) testCases := []struct { name string @@ -125,29 +131,29 @@ func TestCreateRelayPublicRoutes(t *testing.T) { }{ { name: "relay_txn invalid user id", - req: createGetRelayTxnHTTPRequest(base.Cfg.Global.ServerName, "user:local"), + req: createGetRelayTxnHTTPRequest(cfg.Global.ServerName, "user:local"), wantCode: 400, }, { name: "relay_txn valid user id", - req: createGetRelayTxnHTTPRequest(base.Cfg.Global.ServerName, "@user:local"), + req: createGetRelayTxnHTTPRequest(cfg.Global.ServerName, "@user:local"), wantCode: 200, }, { name: "send_relay invalid user id", - req: createSendRelayTxnHTTPRequest(base.Cfg.Global.ServerName, "123", "user:local"), + req: createSendRelayTxnHTTPRequest(cfg.Global.ServerName, "123", "user:local"), wantCode: 400, }, { name: "send_relay valid user id", - req: createSendRelayTxnHTTPRequest(base.Cfg.Global.ServerName, "123", "@user:local"), + req: createSendRelayTxnHTTPRequest(cfg.Global.ServerName, "123", "@user:local"), wantCode: 200, }, } for _, tc := range testCases { w := httptest.NewRecorder() - base.Routers.Federation.ServeHTTP(w, tc.req) + routers.Federation.ServeHTTP(w, tc.req) if w.Code != tc.wantCode { t.Fatalf("%s: got HTTP %d want %d", tc.name, w.Code, tc.wantCode) } @@ -157,16 +163,19 @@ func TestCreateRelayPublicRoutes(t *testing.T) { func TestDisableRelayPublicRoutes(t *testing.T) { test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { - base, close := testrig.CreateBaseDendrite(t, dbType) + cfg, processCtx, close := testrig.CreateConfig(t, dbType) defer close() - caches := caching.NewRistrettoCache(base.Cfg.Global.Cache.EstimatedMaxSize, base.Cfg.Global.Cache.MaxAge, caching.DisableMetrics) + routers := httputil.NewRouters() + caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) + + cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) - relayAPI := relayapi.NewRelayInternalAPI(base, nil, nil, nil, nil, false, caches) + relayAPI := relayapi.NewRelayInternalAPI(cfg, cm, nil, nil, nil, nil, false, caches) assert.NotNil(t, relayAPI) serverKeyAPI := &signing.YggdrasilKeys{} keyRing := serverKeyAPI.KeyRing() - relayapi.AddPublicRoutes(base, keyRing, relayAPI) + relayapi.AddPublicRoutes(routers, cfg, keyRing, relayAPI) testCases := []struct { name string @@ -175,19 +184,19 @@ func TestDisableRelayPublicRoutes(t *testing.T) { }{ { name: "relay_txn valid user id", - req: createGetRelayTxnHTTPRequest(base.Cfg.Global.ServerName, "@user:local"), + req: createGetRelayTxnHTTPRequest(cfg.Global.ServerName, "@user:local"), wantCode: 404, }, { name: "send_relay valid user id", - req: createSendRelayTxnHTTPRequest(base.Cfg.Global.ServerName, "123", "@user:local"), + req: createSendRelayTxnHTTPRequest(cfg.Global.ServerName, "123", "@user:local"), wantCode: 404, }, } for _, tc := range testCases { w := httptest.NewRecorder() - base.Routers.Federation.ServeHTTP(w, tc.req) + routers.Federation.ServeHTTP(w, tc.req) if w.Code != tc.wantCode { t.Fatalf("%s: got HTTP %d want %d", tc.name, w.Code, tc.wantCode) } diff --git a/roomserver/internal/alias.go b/roomserver/internal/alias.go index fc61b7f4a2..94b8b16cf9 100644 --- a/roomserver/internal/alias.go +++ b/roomserver/internal/alias.go @@ -117,7 +117,7 @@ func (r *RoomserverInternalAPI) RemoveRoomAlias( request *api.RemoveRoomAliasRequest, response *api.RemoveRoomAliasResponse, ) error { - _, virtualHost, err := r.Cfg.Matrix.SplitLocalID('@', request.UserID) + _, virtualHost, err := r.Cfg.Global.SplitLocalID('@', request.UserID) if err != nil { return err } @@ -175,12 +175,12 @@ func (r *RoomserverInternalAPI) RemoveRoomAlias( sender = ev.Sender() } - _, senderDomain, err := r.Cfg.Matrix.SplitLocalID('@', sender) + _, senderDomain, err := r.Cfg.Global.SplitLocalID('@', sender) if err != nil { return err } - identity, err := r.Cfg.Matrix.SigningIdentityFor(senderDomain) + identity, err := r.Cfg.Global.SigningIdentityFor(senderDomain) if err != nil { return err } @@ -206,7 +206,7 @@ func (r *RoomserverInternalAPI) RemoveRoomAlias( return err } - newEvent, err := eventutil.BuildEvent(ctx, builder, r.Cfg.Matrix, identity, time.Now(), &eventsNeeded, stateRes) + newEvent, err := eventutil.BuildEvent(ctx, builder, &r.Cfg.Global, identity, time.Now(), &eventsNeeded, stateRes) if err != nil { return err } diff --git a/roomserver/internal/api.go b/roomserver/internal/api.go index 2e987d681e..7ca3675da6 100644 --- a/roomserver/internal/api.go +++ b/roomserver/internal/api.go @@ -18,7 +18,6 @@ import ( "github.com/matrix-org/dendrite/roomserver/internal/query" "github.com/matrix-org/dendrite/roomserver/producers" "github.com/matrix-org/dendrite/roomserver/storage" - "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/jetstream" "github.com/matrix-org/dendrite/setup/process" @@ -41,9 +40,8 @@ type RoomserverInternalAPI struct { *perform.Upgrader *perform.Admin ProcessContext *process.ProcessContext - Base *base.BaseDendrite DB storage.Database - Cfg *config.RoomServer + Cfg *config.Dendrite Cache caching.RoomServerCaches ServerName gomatrixserverlib.ServerName KeyRing gomatrixserverlib.JSONVerifier @@ -56,43 +54,44 @@ type RoomserverInternalAPI struct { InputRoomEventTopic string // JetStream topic for new input room events OutputProducer *producers.RoomEventProducer PerspectiveServerNames []gomatrixserverlib.ServerName + enableMetrics bool } func NewRoomserverAPI( - base *base.BaseDendrite, roomserverDB storage.Database, - js nats.JetStreamContext, nc *nats.Conn, caches caching.RoomServerCaches, + processContext *process.ProcessContext, dendriteCfg *config.Dendrite, roomserverDB storage.Database, + js nats.JetStreamContext, nc *nats.Conn, caches caching.RoomServerCaches, enableMetrics bool, ) *RoomserverInternalAPI { var perspectiveServerNames []gomatrixserverlib.ServerName - for _, kp := range base.Cfg.FederationAPI.KeyPerspectives { + for _, kp := range dendriteCfg.FederationAPI.KeyPerspectives { perspectiveServerNames = append(perspectiveServerNames, kp.ServerName) } serverACLs := acls.NewServerACLs(roomserverDB) producer := &producers.RoomEventProducer{ - Topic: string(base.Cfg.Global.JetStream.Prefixed(jetstream.OutputRoomEvent)), + Topic: string(dendriteCfg.Global.JetStream.Prefixed(jetstream.OutputRoomEvent)), JetStream: js, ACLs: serverACLs, } a := &RoomserverInternalAPI{ - ProcessContext: base.ProcessContext, + ProcessContext: processContext, DB: roomserverDB, - Base: base, - Cfg: &base.Cfg.RoomServer, + Cfg: dendriteCfg, Cache: caches, - ServerName: base.Cfg.Global.ServerName, + ServerName: dendriteCfg.Global.ServerName, PerspectiveServerNames: perspectiveServerNames, - InputRoomEventTopic: base.Cfg.Global.JetStream.Prefixed(jetstream.InputRoomEvent), + InputRoomEventTopic: dendriteCfg.Global.JetStream.Prefixed(jetstream.InputRoomEvent), OutputProducer: producer, JetStream: js, NATSClient: nc, - Durable: base.Cfg.Global.JetStream.Durable("RoomserverInputConsumer"), + Durable: dendriteCfg.Global.JetStream.Durable("RoomserverInputConsumer"), ServerACLs: serverACLs, Queryer: &query.Queryer{ DB: roomserverDB, Cache: caches, - IsLocalServerName: base.Cfg.Global.IsLocalServerName, + IsLocalServerName: dendriteCfg.Global.IsLocalServerName, ServerACLs: serverACLs, }, + enableMetrics: enableMetrics, // perform-er structs get initialised when we have a federation sender to use } return a @@ -105,15 +104,14 @@ func (r *RoomserverInternalAPI) SetFederationAPI(fsAPI fsAPI.RoomserverFederatio r.fsAPI = fsAPI r.KeyRing = keyRing - identity, err := r.Cfg.Matrix.SigningIdentityFor(r.ServerName) + identity, err := r.Cfg.Global.SigningIdentityFor(r.ServerName) if err != nil { logrus.Panic(err) } r.Inputer = &input.Inputer{ - Cfg: &r.Base.Cfg.RoomServer, - Base: r.Base, - ProcessContext: r.Base.ProcessContext, + Cfg: &r.Cfg.RoomServer, + ProcessContext: r.ProcessContext, DB: r.DB, InputRoomEventTopic: r.InputRoomEventTopic, OutputProducer: r.OutputProducer, @@ -129,12 +127,12 @@ func (r *RoomserverInternalAPI) SetFederationAPI(fsAPI fsAPI.RoomserverFederatio } r.Inviter = &perform.Inviter{ DB: r.DB, - Cfg: r.Cfg, + Cfg: &r.Cfg.RoomServer, FSAPI: r.fsAPI, Inputer: r.Inputer, } r.Joiner = &perform.Joiner{ - Cfg: r.Cfg, + Cfg: &r.Cfg.RoomServer, DB: r.DB, FSAPI: r.fsAPI, RSAPI: r, @@ -143,7 +141,7 @@ func (r *RoomserverInternalAPI) SetFederationAPI(fsAPI fsAPI.RoomserverFederatio } r.Peeker = &perform.Peeker{ ServerName: r.ServerName, - Cfg: r.Cfg, + Cfg: &r.Cfg.RoomServer, DB: r.DB, FSAPI: r.fsAPI, Inputer: r.Inputer, @@ -154,12 +152,12 @@ func (r *RoomserverInternalAPI) SetFederationAPI(fsAPI fsAPI.RoomserverFederatio } r.Unpeeker = &perform.Unpeeker{ ServerName: r.ServerName, - Cfg: r.Cfg, + Cfg: &r.Cfg.RoomServer, FSAPI: r.fsAPI, Inputer: r.Inputer, } r.Leaver = &perform.Leaver{ - Cfg: r.Cfg, + Cfg: &r.Cfg.RoomServer, DB: r.DB, FSAPI: r.fsAPI, Inputer: r.Inputer, @@ -168,7 +166,7 @@ func (r *RoomserverInternalAPI) SetFederationAPI(fsAPI fsAPI.RoomserverFederatio DB: r.DB, } r.Backfiller = &perform.Backfiller{ - IsLocalServerName: r.Cfg.Matrix.IsLocalServerName, + IsLocalServerName: r.Cfg.Global.IsLocalServerName, DB: r.DB, FSAPI: r.fsAPI, KeyRing: r.KeyRing, @@ -181,12 +179,12 @@ func (r *RoomserverInternalAPI) SetFederationAPI(fsAPI fsAPI.RoomserverFederatio DB: r.DB, } r.Upgrader = &perform.Upgrader{ - Cfg: r.Cfg, + Cfg: &r.Cfg.RoomServer, URSAPI: r, } r.Admin = &perform.Admin{ DB: r.DB, - Cfg: r.Cfg, + Cfg: &r.Cfg.RoomServer, Inputer: r.Inputer, Queryer: r.Queryer, Leaver: r.Leaver, diff --git a/roomserver/internal/helpers/helpers_test.go b/roomserver/internal/helpers/helpers_test.go index 03a8bf5752..dd74b844a0 100644 --- a/roomserver/internal/helpers/helpers_test.go +++ b/roomserver/internal/helpers/helpers_test.go @@ -19,7 +19,7 @@ import ( func mustCreateDatabase(t *testing.T, dbType test.DBType) (storage.Database, func()) { conStr, close := test.PrepareDBConnectionString(t, dbType) caches := caching.NewRistrettoCache(8*1024*1024, time.Hour, caching.DisableMetrics) - cm := sqlutil.NewConnectionManager() + cm := sqlutil.NewConnectionManager(nil, config.DatabaseOptions{}) db, err := storage.Open(context.Background(), cm, &config.DatabaseOptions{ConnectionString: config.DataSource(conStr)}, caches) if err != nil { t.Fatalf("failed to create Database: %v", err) diff --git a/roomserver/internal/input/input.go b/roomserver/internal/input/input.go index 2ec19f010c..cc0c673d02 100644 --- a/roomserver/internal/input/input.go +++ b/roomserver/internal/input/input.go @@ -39,7 +39,6 @@ import ( "github.com/matrix-org/dendrite/roomserver/producers" "github.com/matrix-org/dendrite/roomserver/storage" "github.com/matrix-org/dendrite/roomserver/types" - "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/jetstream" "github.com/matrix-org/dendrite/setup/process" @@ -74,7 +73,6 @@ import ( // or C. type Inputer struct { Cfg *config.RoomServer - Base *base.BaseDendrite ProcessContext *process.ProcessContext DB storage.RoomDatabase NATSClient *nats.Conn @@ -89,8 +87,9 @@ type Inputer struct { OutputProducer *producers.RoomEventProducer workers sync.Map // room ID -> *worker - Queryer *query.Queryer - UserAPI userapi.RoomserverUserAPI + Queryer *query.Queryer + UserAPI userapi.RoomserverUserAPI + enableMetrics bool } // If a room consumer is inactive for a while then we will allow NATS @@ -177,7 +176,7 @@ func (r *Inputer) startWorkerForRoom(roomID string) { // will look to see if we have a worker for that room which has its // own consumer. If we don't, we'll start one. func (r *Inputer) Start() error { - if r.Base.EnableMetrics { + if r.enableMetrics { prometheus.MustRegister(roomserverInputBackpressure, processRoomEventDuration) } _, err := r.JetStream.Subscribe( diff --git a/roomserver/internal/input/input_test.go b/roomserver/internal/input/input_test.go index 555ec9c6dc..51c50c37a5 100644 --- a/roomserver/internal/input/input_test.go +++ b/roomserver/internal/input/input_test.go @@ -2,84 +2,69 @@ package input_test import ( "context" - "os" "testing" "time" "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/roomserver" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/roomserver/internal/input" - "github.com/matrix-org/dendrite/roomserver/storage" - "github.com/matrix-org/dendrite/setup/base" - "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/dendrite/setup/jetstream" + "github.com/matrix-org/dendrite/test" "github.com/matrix-org/dendrite/test/testrig" "github.com/matrix-org/gomatrixserverlib" - "github.com/nats-io/nats.go" ) -var js nats.JetStreamContext -var jc *nats.Conn +func TestSingleTransactionOnInput(t *testing.T) { + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + cfg, processCtx, close := testrig.CreateConfig(t, dbType) + defer close() + cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) -func TestMain(m *testing.M) { - var b *base.BaseDendrite - b, js, jc = testrig.Base(nil) - code := m.Run() - b.ShutdownDendrite() - b.WaitForComponentsToFinish() - os.Exit(code) -} + natsInstance := &jetstream.NATSInstance{} + js, jc := natsInstance.Prepare(processCtx, &cfg.Global.JetStream) + caches := caching.NewRistrettoCache(8*1024*1024, time.Hour, caching.DisableMetrics) + rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, natsInstance, caches, caching.DisableMetrics) + rsAPI.SetFederationAPI(nil, nil) -func TestSingleTransactionOnInput(t *testing.T) { - deadline, _ := t.Deadline() - if max := time.Now().Add(time.Second * 3); deadline.After(max) { - deadline = max - } - ctx, cancel := context.WithDeadline(context.Background(), deadline) - defer cancel() + deadline, _ := t.Deadline() + if max := time.Now().Add(time.Second * 3); deadline.Before(max) { + deadline = max + } + ctx, cancel := context.WithDeadline(processCtx.Context(), deadline) + defer cancel() + + event, err := gomatrixserverlib.NewEventFromTrustedJSON( + []byte(`{"auth_events":[],"content":{"creator":"@neilalexander:dendrite.matrix.org","room_version":"6"},"depth":1,"hashes":{"sha256":"jqOqdNEH5r0NiN3xJtj0u5XUVmRqq9YvGbki1wxxuuM"},"origin":"dendrite.matrix.org","origin_server_ts":1644595362726,"prev_events":[],"prev_state":[],"room_id":"!jSZZRknA6GkTBXNP:dendrite.matrix.org","sender":"@neilalexander:dendrite.matrix.org","signatures":{"dendrite.matrix.org":{"ed25519:6jB2aB":"bsQXO1wketf1OSe9xlndDIWe71W9KIundc6rBw4KEZdGPW7x4Tv4zDWWvbxDsG64sS2IPWfIm+J0OOozbrWIDw"}},"state_key":"","type":"m.room.create"}`), + false, gomatrixserverlib.RoomVersionV6, + ) + if err != nil { + t.Fatal(err) + } + in := api.InputRoomEvent{ + Kind: api.KindOutlier, // don't panic if we generate an output event + Event: event.Headered(gomatrixserverlib.RoomVersionV6), + } - event, err := gomatrixserverlib.NewEventFromTrustedJSON( - []byte(`{"auth_events":[],"content":{"creator":"@neilalexander:dendrite.matrix.org","room_version":"6"},"depth":1,"hashes":{"sha256":"jqOqdNEH5r0NiN3xJtj0u5XUVmRqq9YvGbki1wxxuuM"},"origin":"dendrite.matrix.org","origin_server_ts":1644595362726,"prev_events":[],"prev_state":[],"room_id":"!jSZZRknA6GkTBXNP:dendrite.matrix.org","sender":"@neilalexander:dendrite.matrix.org","signatures":{"dendrite.matrix.org":{"ed25519:6jB2aB":"bsQXO1wketf1OSe9xlndDIWe71W9KIundc6rBw4KEZdGPW7x4Tv4zDWWvbxDsG64sS2IPWfIm+J0OOozbrWIDw"}},"state_key":"","type":"m.room.create"}`), - false, gomatrixserverlib.RoomVersionV6, - ) - if err != nil { - t.Fatal(err) - } - in := api.InputRoomEvent{ - Kind: api.KindOutlier, // don't panic if we generate an output event - Event: event.Headered(gomatrixserverlib.RoomVersionV6), - } - cm := sqlutil.NewConnectionManager() - db, err := storage.Open( - context.Background(), cm, - &config.DatabaseOptions{ - ConnectionString: "", - MaxOpenConnections: 1, - MaxIdleConnections: 1, - }, - caching.NewRistrettoCache(8*1024*1024, time.Hour, caching.DisableMetrics), - ) - if err != nil { - t.Logf("PostgreSQL not available (%s), skipping", err) - t.SkipNow() - } - inputter := &input.Inputer{ - DB: db, - JetStream: js, - NATSClient: jc, - } - res := &api.InputRoomEventsResponse{} - inputter.InputRoomEvents( - ctx, - &api.InputRoomEventsRequest{ - InputRoomEvents: []api.InputRoomEvent{in}, - Asynchronous: false, - }, - res, - ) - // If we fail here then it's because we've hit the test deadline, - // so we probably deadlocked - if err := res.Err(); err != nil { - t.Fatal(err) - } + inputter := &input.Inputer{ + JetStream: js, + NATSClient: jc, + Cfg: &cfg.RoomServer, + } + res := &api.InputRoomEventsResponse{} + inputter.InputRoomEvents( + ctx, + &api.InputRoomEventsRequest{ + InputRoomEvents: []api.InputRoomEvent{in}, + Asynchronous: false, + }, + res, + ) + // If we fail here then it's because we've hit the test deadline, + // so we probably deadlocked + if err := res.Err(); err != nil { + t.Fatal(err) + } + }) } diff --git a/roomserver/roomserver.go b/roomserver/roomserver.go index 1c55423ef1..4685f474f3 100644 --- a/roomserver/roomserver.go +++ b/roomserver/roomserver.go @@ -16,29 +16,34 @@ package roomserver import ( "github.com/matrix-org/dendrite/internal/caching" + "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/dendrite/setup/jetstream" + "github.com/matrix-org/dendrite/setup/process" "github.com/sirupsen/logrus" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/roomserver/internal" "github.com/matrix-org/dendrite/roomserver/storage" - "github.com/matrix-org/dendrite/setup/base" ) // NewInternalAPI returns a concrete implementation of the internal API. func NewInternalAPI( - base *base.BaseDendrite, + processContext *process.ProcessContext, + cfg *config.Dendrite, + cm sqlutil.Connections, + natsInstance *jetstream.NATSInstance, caches caching.RoomServerCaches, + enableMetrics bool, ) api.RoomserverInternalAPI { - cfg := &base.Cfg.RoomServer - - roomserverDB, err := storage.Open(base.ProcessContext.Context(), base.ConnectionManager, &cfg.Database, caches) + roomserverDB, err := storage.Open(processContext.Context(), cm, &cfg.RoomServer.Database, caches) if err != nil { logrus.WithError(err).Panicf("failed to connect to room server db") } - js, nc := base.NATS.Prepare(base.ProcessContext, &cfg.Matrix.JetStream) + js, nc := natsInstance.Prepare(processContext, &cfg.Global.JetStream) return internal.NewRoomserverAPI( - base, roomserverDB, js, nc, caches, + processContext, cfg, roomserverDB, js, nc, caches, enableMetrics, ) } diff --git a/roomserver/roomserver_test.go b/roomserver/roomserver_test.go index 1b0b3155db..729da15b3f 100644 --- a/roomserver/roomserver_test.go +++ b/roomserver/roomserver_test.go @@ -8,11 +8,12 @@ import ( "time" "github.com/matrix-org/dendrite/internal/caching" + "github.com/matrix-org/dendrite/internal/httputil" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/stretchr/testify/assert" "github.com/matrix-org/dendrite/roomserver/state" "github.com/matrix-org/dendrite/roomserver/types" - "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/userapi" userAPI "github.com/matrix-org/dendrite/userapi/api" @@ -30,23 +31,14 @@ import ( "github.com/matrix-org/dendrite/test/testrig" ) -func mustCreateDatabase(t *testing.T, dbType test.DBType) (*base.BaseDendrite, storage.Database, func()) { - t.Helper() - base, close := testrig.CreateBaseDendrite(t, dbType) - caches := caching.NewRistrettoCache(base.Cfg.Global.Cache.EstimatedMaxSize, base.Cfg.Global.Cache.MaxAge, caching.DisableMetrics) - db, err := storage.Open(base.ProcessContext.Context(), base.ConnectionManager, &base.Cfg.RoomServer.Database, caches) - if err != nil { - t.Fatalf("failed to create Database: %v", err) - } - return base, db, close -} - func TestUsers(t *testing.T) { test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { - base, close := testrig.CreateBaseDendrite(t, dbType) + cfg, processCtx, close := testrig.CreateConfig(t, dbType) defer close() - caches := caching.NewRistrettoCache(base.Cfg.Global.Cache.EstimatedMaxSize, base.Cfg.Global.Cache.MaxAge, caching.DisableMetrics) - rsAPI := roomserver.NewInternalAPI(base, caches) + caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) + natsInstance := jetstream.NATSInstance{} + cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) + rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) // SetFederationAPI starts the room event input consumer rsAPI.SetFederationAPI(nil, nil) @@ -55,7 +47,7 @@ func TestUsers(t *testing.T) { }) t.Run("kick users", func(t *testing.T) { - usrAPI := userapi.NewInternalAPI(base, rsAPI, nil) + usrAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) rsAPI.SetUserAPI(usrAPI) testKickUsers(t, rsAPI, usrAPI) }) @@ -181,11 +173,13 @@ func Test_QueryLeftUsers(t *testing.T) { ctx := context.Background() test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { - base, _, close := mustCreateDatabase(t, dbType) + cfg, processCtx, close := testrig.CreateConfig(t, dbType) defer close() - caches := caching.NewRistrettoCache(base.Cfg.Global.Cache.EstimatedMaxSize, base.Cfg.Global.Cache.MaxAge, caching.DisableMetrics) - rsAPI := roomserver.NewInternalAPI(base, caches) + caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) + natsInstance := jetstream.NATSInstance{} + cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) + rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) // SetFederationAPI starts the room event input consumer rsAPI.SetFederationAPI(nil, nil) // Create the room @@ -229,30 +223,35 @@ func TestPurgeRoom(t *testing.T) { ctx := context.Background() test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { - base, db, close := mustCreateDatabase(t, dbType) + cfg, processCtx, close := testrig.CreateConfig(t, dbType) + natsInstance := jetstream.NATSInstance{} defer close() + routers := httputil.NewRouters() + cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) + caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) + db, err := storage.Open(processCtx.Context(), cm, &cfg.RoomServer.Database, caches) + if err != nil { + t.Fatal(err) + } + jsCtx, _ := natsInstance.Prepare(processCtx, &cfg.Global.JetStream) + defer jetstream.DeleteAllStreams(jsCtx, &cfg.Global.JetStream) - jsCtx, _ := base.NATS.Prepare(base.ProcessContext, &base.Cfg.Global.JetStream) - defer jetstream.DeleteAllStreams(jsCtx, &base.Cfg.Global.JetStream) - - fedClient := base.CreateFederationClient() - caches := caching.NewRistrettoCache(base.Cfg.Global.Cache.EstimatedMaxSize, base.Cfg.Global.Cache.MaxAge, caching.DisableMetrics) - rsAPI := roomserver.NewInternalAPI(base, caches) - userAPI := userapi.NewInternalAPI(base, rsAPI, nil) + rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) // this starts the JetStream consumers - syncapi.AddPublicRoutes(base, userAPI, rsAPI, caches) - federationapi.NewInternalAPI(base, fedClient, rsAPI, caches, nil, true) + syncapi.AddPublicRoutes(processCtx, routers, cfg, cm, &natsInstance, userAPI, rsAPI, caches, caching.DisableMetrics) + federationapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, rsAPI, caches, nil, true) rsAPI.SetFederationAPI(nil, nil) // Create the room - if err := api.SendEvents(ctx, rsAPI, api.KindNew, room.Events(), "test", "test", "test", nil, false); err != nil { + if err = api.SendEvents(ctx, rsAPI, api.KindNew, room.Events(), "test", "test", "test", nil, false); err != nil { t.Fatalf("failed to send events: %v", err) } // some dummy entries to validate after purging publishResp := &api.PerformPublishResponse{} - if err := rsAPI.PerformPublish(ctx, &api.PerformPublishRequest{RoomID: room.ID, Visibility: "public"}, publishResp); err != nil { + if err = rsAPI.PerformPublish(ctx, &api.PerformPublishRequest{RoomID: room.ID, Visibility: "public"}, publishResp); err != nil { t.Fatal(err) } if publishResp.Error != nil { @@ -340,7 +339,7 @@ func TestPurgeRoom(t *testing.T) { t.Fatalf("test timed out after %s", timeout) } sum = 0 - consumerCh := jsCtx.Consumers(base.Cfg.Global.JetStream.Prefixed(jetstream.OutputRoomEvent)) + consumerCh := jsCtx.Consumers(cfg.Global.JetStream.Prefixed(jetstream.OutputRoomEvent)) for x := range consumerCh { sum += x.NumAckPending } @@ -508,8 +507,14 @@ func TestRedaction(t *testing.T) { ctx := context.Background() test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { - _, db, close := mustCreateDatabase(t, dbType) + cfg, processCtx, close := testrig.CreateConfig(t, dbType) defer close() + cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) + caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) + db, err := storage.Open(processCtx.Context(), cm, &cfg.RoomServer.Database, caches) + if err != nil { + t.Fatal(err) + } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { diff --git a/roomserver/storage/shared/storage_test.go b/roomserver/storage/shared/storage_test.go index 684e80b8f3..941e848021 100644 --- a/roomserver/storage/shared/storage_test.go +++ b/roomserver/storage/shared/storage_test.go @@ -15,14 +15,12 @@ import ( "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/test" - "github.com/matrix-org/dendrite/test/testrig" ) func mustCreateRoomserverDatabase(t *testing.T, dbType test.DBType) (*shared.Database, func()) { t.Helper() connStr, clearDB := test.PrepareDBConnectionString(t, dbType) - base, _, _ := testrig.Base(nil) dbOpts := &config.DatabaseOptions{ConnectionString: config.DataSource(connStr)} db, err := sqlutil.Open(dbOpts, sqlutil.NewExclusiveWriter()) @@ -61,8 +59,6 @@ func mustCreateRoomserverDatabase(t *testing.T, dbType test.DBType) (*shared.Dat Writer: sqlutil.NewExclusiveWriter(), Cache: cache, }, func() { - err := base.Close() - assert.NoError(t, err) clearDB() err = db.Close() assert.NoError(t, err) diff --git a/setup/base/base.go b/setup/base/base.go index 8c9b06d0e9..e5fd6fed11 100644 --- a/setup/base/base.go +++ b/setup/base/base.go @@ -22,30 +22,24 @@ import ( "errors" "fmt" "html/template" - "io" "io/fs" "net" "net/http" _ "net/http/pprof" "os" "os/signal" - "sync" "syscall" "time" - "github.com/getsentry/sentry-go" sentryhttp "github.com/getsentry/sentry-go/http" - "github.com/matrix-org/dendrite/setup/jetstream" "github.com/matrix-org/gomatrixserverlib" "github.com/prometheus/client_golang/prometheus/promhttp" "go.uber.org/atomic" - "github.com/matrix-org/dendrite/internal" - "github.com/matrix-org/dendrite/internal/httputil" - "github.com/matrix-org/dendrite/internal/sqlutil" - "github.com/gorilla/mux" "github.com/kardianos/minwinsvc" + "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/httputil" "github.com/sirupsen/logrus" @@ -56,154 +50,22 @@ import ( //go:embed static/*.gotmpl var staticContent embed.FS -// BaseDendrite is a base for creating new instances of dendrite. It parses -// command line flags and config, and exposes methods for creating various -// resources. All errors are handled by logging then exiting, so all methods -// should only be used during start up. -// Must be closed when shutting down. -type BaseDendrite struct { - *process.ProcessContext - tracerCloser io.Closer - Routers httputil.Routers - NATS *jetstream.NATSInstance - Cfg *config.Dendrite - DNSCache *gomatrixserverlib.DNSCache - ConnectionManager sqlutil.Connections - EnableMetrics bool - startupLock sync.Mutex -} - const HTTPServerTimeout = time.Minute * 5 -type BaseDendriteOptions int - -const ( - DisableMetrics BaseDendriteOptions = iota -) - -// NewBaseDendrite creates a new instance to be used by a component. -func NewBaseDendrite(cfg *config.Dendrite, options ...BaseDendriteOptions) *BaseDendrite { - platformSanityChecks() - enableMetrics := true - for _, opt := range options { - switch opt { - case DisableMetrics: - enableMetrics = false - } - } - - configErrors := &config.ConfigErrors{} - cfg.Verify(configErrors) - if len(*configErrors) > 0 { - for _, err := range *configErrors { - logrus.Errorf("Configuration error: %s", err) - } - logrus.Fatalf("Failed to start due to configuration errors") - } - - internal.SetupStdLogging() - internal.SetupHookLogging(cfg.Logging) - internal.SetupPprof() - - logrus.Infof("Dendrite version %s", internal.VersionString()) - - if !cfg.ClientAPI.RegistrationDisabled && cfg.ClientAPI.OpenRegistrationWithoutVerificationEnabled { - logrus.Warn("Open registration is enabled") - } - - closer, err := cfg.SetupTracing() - if err != nil { - logrus.WithError(err).Panicf("failed to start opentracing") - } - - if cfg.Global.Sentry.Enabled { - logrus.Info("Setting up Sentry for debugging...") - err = sentry.Init(sentry.ClientOptions{ - Dsn: cfg.Global.Sentry.DSN, - Environment: cfg.Global.Sentry.Environment, - Debug: true, - ServerName: string(cfg.Global.ServerName), - Release: "dendrite@" + internal.VersionString(), - AttachStacktrace: true, - }) - if err != nil { - logrus.WithError(err).Panic("failed to start Sentry") - } - } - - var dnsCache *gomatrixserverlib.DNSCache - if cfg.Global.DNSCache.Enabled { - dnsCache = gomatrixserverlib.NewDNSCache( - cfg.Global.DNSCache.CacheSize, - cfg.Global.DNSCache.CacheLifetime, - ) - logrus.Infof( - "DNS cache enabled (size %d, lifetime %s)", - cfg.Global.DNSCache.CacheSize, - cfg.Global.DNSCache.CacheLifetime, - ) - } - - // If we're in monolith mode, we'll set up a global pool of database - // connections. A component is welcome to use this pool if they don't - // have a separate database config of their own. - cm := sqlutil.NewConnectionManager() - if cfg.Global.DatabaseOptions.ConnectionString != "" { - if cfg.Global.DatabaseOptions.ConnectionString.IsSQLite() { - logrus.Panic("Using a global database connection pool is not supported with SQLite databases") - } - _, _, err := cm.Connection(&cfg.Global.DatabaseOptions) - if err != nil { - logrus.WithError(err).Panic("Failed to set up global database connections") - } - logrus.Debug("Using global database connection pool") - } - - // Ideally we would only use SkipClean on routes which we know can allow '/' but due to - // https://github.com/gorilla/mux/issues/460 we have to attach this at the top router. - // When used in conjunction with UseEncodedPath() we get the behaviour we want when parsing - // path parameters: - // /foo/bar%2Fbaz == [foo, bar%2Fbaz] (from UseEncodedPath) - // /foo/bar%2F%2Fbaz == [foo, bar%2F%2Fbaz] (from SkipClean) - // In particular, rooms v3 event IDs are not urlsafe and can include '/' and because they - // are randomly generated it results in flakey tests. - // We need to be careful with media APIs if they read from a filesystem to make sure they - // are not inadvertently reading paths without cleaning, else this could introduce a - // directory traversal attack e.g /../../../etc/passwd - - return &BaseDendrite{ - ProcessContext: process.NewProcessContext(), - tracerCloser: closer, - Cfg: cfg, - DNSCache: dnsCache, - Routers: httputil.NewRouters(), - NATS: &jetstream.NATSInstance{}, - ConnectionManager: cm, - EnableMetrics: enableMetrics, - } -} - -// Close implements io.Closer -func (b *BaseDendrite) Close() error { - b.ProcessContext.ShutdownDendrite() - b.ProcessContext.WaitForShutdown() - return b.tracerCloser.Close() -} - // CreateClient creates a new client (normally used for media fetch requests). // Should only be called once per component. -func (b *BaseDendrite) CreateClient() *gomatrixserverlib.Client { - if b.Cfg.Global.DisableFederation { +func CreateClient(cfg *config.Dendrite, dnsCache *gomatrixserverlib.DNSCache) *gomatrixserverlib.Client { + if cfg.Global.DisableFederation { return gomatrixserverlib.NewClient( gomatrixserverlib.WithTransport(noOpHTTPTransport), ) } opts := []gomatrixserverlib.ClientOption{ - gomatrixserverlib.WithSkipVerify(b.Cfg.FederationAPI.DisableTLSValidation), + gomatrixserverlib.WithSkipVerify(cfg.FederationAPI.DisableTLSValidation), gomatrixserverlib.WithWellKnownSRVLookups(true), } - if b.Cfg.Global.DNSCache.Enabled { - opts = append(opts, gomatrixserverlib.WithDNSCache(b.DNSCache)) + if cfg.Global.DNSCache.Enabled && dnsCache != nil { + opts = append(opts, gomatrixserverlib.WithDNSCache(dnsCache)) } client := gomatrixserverlib.NewClient(opts...) client.SetUserAgent(fmt.Sprintf("Dendrite/%s", internal.VersionString())) @@ -212,20 +74,20 @@ func (b *BaseDendrite) CreateClient() *gomatrixserverlib.Client { // CreateFederationClient creates a new federation client. Should only be called // once per component. -func (b *BaseDendrite) CreateFederationClient() *gomatrixserverlib.FederationClient { - identities := b.Cfg.Global.SigningIdentities() - if b.Cfg.Global.DisableFederation { +func CreateFederationClient(cfg *config.Dendrite, dnsCache *gomatrixserverlib.DNSCache) *gomatrixserverlib.FederationClient { + identities := cfg.Global.SigningIdentities() + if cfg.Global.DisableFederation { return gomatrixserverlib.NewFederationClient( identities, gomatrixserverlib.WithTransport(noOpHTTPTransport), ) } opts := []gomatrixserverlib.ClientOption{ gomatrixserverlib.WithTimeout(time.Minute * 5), - gomatrixserverlib.WithSkipVerify(b.Cfg.FederationAPI.DisableTLSValidation), - gomatrixserverlib.WithKeepAlives(!b.Cfg.FederationAPI.DisableHTTPKeepalives), + gomatrixserverlib.WithSkipVerify(cfg.FederationAPI.DisableTLSValidation), + gomatrixserverlib.WithKeepAlives(!cfg.FederationAPI.DisableHTTPKeepalives), } - if b.Cfg.Global.DNSCache.Enabled { - opts = append(opts, gomatrixserverlib.WithDNSCache(b.DNSCache)) + if cfg.Global.DNSCache.Enabled { + opts = append(opts, gomatrixserverlib.WithDNSCache(dnsCache)) } client := gomatrixserverlib.NewFederationClient( identities, opts..., @@ -234,12 +96,12 @@ func (b *BaseDendrite) CreateFederationClient() *gomatrixserverlib.FederationCli return client } -func (b *BaseDendrite) ConfigureAdminEndpoints() { - b.Routers.DendriteAdmin.HandleFunc("/monitor/up", func(w http.ResponseWriter, r *http.Request) { +func ConfigureAdminEndpoints(processContext *process.ProcessContext, routers httputil.Routers) { + routers.DendriteAdmin.HandleFunc("/monitor/up", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(200) }) - b.Routers.DendriteAdmin.HandleFunc("/monitor/health", func(w http.ResponseWriter, r *http.Request) { - if isDegraded, reasons := b.ProcessContext.IsDegraded(); isDegraded { + routers.DendriteAdmin.HandleFunc("/monitor/health", func(w http.ResponseWriter, r *http.Request) { + if isDegraded, reasons := processContext.IsDegraded(); isDegraded { w.WriteHeader(503) _ = json.NewEncoder(w).Encode(struct { Warnings []string `json:"warnings"` @@ -254,14 +116,13 @@ func (b *BaseDendrite) ConfigureAdminEndpoints() { // SetupAndServeHTTP sets up the HTTP server to serve client & federation APIs // and adds a prometheus handler under /_dendrite/metrics. -func (b *BaseDendrite) SetupAndServeHTTP( +func SetupAndServeHTTP( + processContext *process.ProcessContext, + cfg *config.Dendrite, + routers httputil.Routers, externalHTTPAddr config.ServerAddress, certFile, keyFile *string, ) { - // Manually unlocked right before actually serving requests, - // as we don't return from this method (defer doesn't work). - b.startupLock.Lock() - externalRouter := mux.NewRouter().SkipClean(true).UseEncodedPath() externalServ := &http.Server{ @@ -269,7 +130,7 @@ func (b *BaseDendrite) SetupAndServeHTTP( WriteTimeout: HTTPServerTimeout, Handler: externalRouter, BaseContext: func(_ net.Listener) context.Context { - return b.ProcessContext.Context() + return processContext.Context() }, } @@ -278,11 +139,11 @@ func (b *BaseDendrite) SetupAndServeHTTP( http.Redirect(w, r, httputil.PublicStaticPath, http.StatusFound) }) - if b.Cfg.Global.Metrics.Enabled { - externalRouter.Handle("/metrics", httputil.WrapHandlerInBasicAuth(promhttp.Handler(), b.Cfg.Global.Metrics.BasicAuth)) + if cfg.Global.Metrics.Enabled { + externalRouter.Handle("/metrics", httputil.WrapHandlerInBasicAuth(promhttp.Handler(), cfg.Global.Metrics.BasicAuth)) } - b.ConfigureAdminEndpoints() + ConfigureAdminEndpoints(processContext, routers) // Parse and execute the landing page template tmpl := template.Must(template.ParseFS(staticContent, "static/*.gotmpl")) @@ -293,38 +154,36 @@ func (b *BaseDendrite) SetupAndServeHTTP( logrus.WithError(err).Fatal("failed to execute landing page template") } - b.Routers.Static.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + routers.Static.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write(landingPage.Bytes()) }) var clientHandler http.Handler - clientHandler = b.Routers.Client - if b.Cfg.Global.Sentry.Enabled { + clientHandler = routers.Client + if cfg.Global.Sentry.Enabled { sentryHandler := sentryhttp.New(sentryhttp.Options{ Repanic: true, }) - clientHandler = sentryHandler.Handle(b.Routers.Client) + clientHandler = sentryHandler.Handle(routers.Client) } var federationHandler http.Handler - federationHandler = b.Routers.Federation - if b.Cfg.Global.Sentry.Enabled { + federationHandler = routers.Federation + if cfg.Global.Sentry.Enabled { sentryHandler := sentryhttp.New(sentryhttp.Options{ Repanic: true, }) - federationHandler = sentryHandler.Handle(b.Routers.Federation) + federationHandler = sentryHandler.Handle(routers.Federation) } - externalRouter.PathPrefix(httputil.DendriteAdminPathPrefix).Handler(b.Routers.DendriteAdmin) + externalRouter.PathPrefix(httputil.DendriteAdminPathPrefix).Handler(routers.DendriteAdmin) externalRouter.PathPrefix(httputil.PublicClientPathPrefix).Handler(clientHandler) - if !b.Cfg.Global.DisableFederation { - externalRouter.PathPrefix(httputil.PublicKeyPathPrefix).Handler(b.Routers.Keys) + if !cfg.Global.DisableFederation { + externalRouter.PathPrefix(httputil.PublicKeyPathPrefix).Handler(routers.Keys) externalRouter.PathPrefix(httputil.PublicFederationPathPrefix).Handler(federationHandler) } - externalRouter.PathPrefix(httputil.SynapseAdminPathPrefix).Handler(b.Routers.SynapseAdmin) - externalRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(b.Routers.Media) - externalRouter.PathPrefix(httputil.PublicWellKnownPrefix).Handler(b.Routers.WellKnown) - externalRouter.PathPrefix(httputil.PublicStaticPath).Handler(b.Routers.Static) - - b.startupLock.Unlock() + externalRouter.PathPrefix(httputil.SynapseAdminPathPrefix).Handler(routers.SynapseAdmin) + externalRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(routers.Media) + externalRouter.PathPrefix(httputil.PublicWellKnownPrefix).Handler(routers.WellKnown) + externalRouter.PathPrefix(httputil.PublicStaticPath).Handler(routers.Static) externalRouter.NotFoundHandler = httputil.NotFoundCORSHandler externalRouter.MethodNotAllowedHandler = httputil.NotAllowedHandler @@ -333,10 +192,10 @@ func (b *BaseDendrite) SetupAndServeHTTP( go func() { var externalShutdown atomic.Bool // RegisterOnShutdown can be called more than once logrus.Infof("Starting external listener on %s", externalServ.Addr) - b.ProcessContext.ComponentStarted() + processContext.ComponentStarted() externalServ.RegisterOnShutdown(func() { if externalShutdown.CompareAndSwap(false, true) { - b.ProcessContext.ComponentFinished() + processContext.ComponentFinished() logrus.Infof("Stopped external HTTP listener") } }) @@ -378,32 +237,27 @@ func (b *BaseDendrite) SetupAndServeHTTP( }() } - minwinsvc.SetOnExit(b.ProcessContext.ShutdownDendrite) - <-b.ProcessContext.WaitForShutdown() + minwinsvc.SetOnExit(processContext.ShutdownDendrite) + <-processContext.WaitForShutdown() logrus.Infof("Stopping HTTP listeners") _ = externalServ.Shutdown(context.Background()) logrus.Infof("Stopped HTTP listeners") } -func (b *BaseDendrite) WaitForShutdown() { +func WaitForShutdown(processCtx *process.ProcessContext) { sigs := make(chan os.Signal, 1) signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM) select { case <-sigs: - case <-b.ProcessContext.WaitForShutdown(): + case <-processCtx.WaitForShutdown(): } signal.Reset(syscall.SIGINT, syscall.SIGTERM) logrus.Warnf("Shutdown signal received") - b.ProcessContext.ShutdownDendrite() - b.ProcessContext.WaitForComponentsToFinish() - if b.Cfg.Global.Sentry.Enabled { - if !sentry.Flush(time.Second * 5) { - logrus.Warnf("failed to flush all Sentry events!") - } - } + processCtx.ShutdownDendrite() + processCtx.WaitForComponentsToFinish() logrus.Warnf("Dendrite is exiting now") } diff --git a/setup/base/base_test.go b/setup/base/base_test.go index aa406db2c1..bba967b941 100644 --- a/setup/base/base_test.go +++ b/setup/base/base_test.go @@ -13,8 +13,10 @@ import ( "time" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/httputil" + basepkg "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/config" - "github.com/matrix-org/dendrite/test/testrig" + "github.com/matrix-org/dendrite/setup/process" "github.com/stretchr/testify/assert" ) @@ -30,8 +32,10 @@ func TestLandingPage_Tcp(t *testing.T) { }) assert.NoError(t, err) - b, _, _ := testrig.Base(nil) - defer b.Close() + processCtx := process.NewProcessContext() + routers := httputil.NewRouters() + cfg := config.Dendrite{} + cfg.Defaults(config.DefaultOpts{Generate: true, SingleDatabase: true}) // hack: create a server and close it immediately, just to get a random port assigned s := httptest.NewServer(nil) @@ -40,7 +44,7 @@ func TestLandingPage_Tcp(t *testing.T) { // start base with the listener and wait for it to be started address, err := config.HTTPAddress(s.URL) assert.NoError(t, err) - go b.SetupAndServeHTTP(address, nil, nil) + go basepkg.SetupAndServeHTTP(processCtx, &cfg, routers, address, nil, nil) time.Sleep(time.Millisecond * 10) // When hitting /, we should be redirected to /_matrix/static, which should contain the landing page @@ -70,15 +74,17 @@ func TestLandingPage_UnixSocket(t *testing.T) { }) assert.NoError(t, err) - b, _, _ := testrig.Base(nil) - defer b.Close() + processCtx := process.NewProcessContext() + routers := httputil.NewRouters() + cfg := config.Dendrite{} + cfg.Defaults(config.DefaultOpts{Generate: true, SingleDatabase: true}) tempDir := t.TempDir() socket := path.Join(tempDir, "socket") // start base with the listener and wait for it to be started address, err := config.UnixSocketAddress(socket, "755") assert.NoError(t, err) - go b.SetupAndServeHTTP(address, nil, nil) + go basepkg.SetupAndServeHTTP(processCtx, &cfg, routers, address, nil, nil) time.Sleep(time.Millisecond * 100) client := &http.Client{ diff --git a/setup/base/sanity_other.go b/setup/base/sanity_other.go index 48fe6e1f85..d35c2e8727 100644 --- a/setup/base/sanity_other.go +++ b/setup/base/sanity_other.go @@ -3,6 +3,6 @@ package base -func platformSanityChecks() { +func PlatformSanityChecks() { // Nothing to do yet. } diff --git a/setup/base/sanity_unix.go b/setup/base/sanity_unix.go index c630d3f193..0403df1a8f 100644 --- a/setup/base/sanity_unix.go +++ b/setup/base/sanity_unix.go @@ -9,7 +9,7 @@ import ( "github.com/sirupsen/logrus" ) -func platformSanityChecks() { +func PlatformSanityChecks() { // Dendrite needs a relatively high number of file descriptors in order // to function properly, particularly when federating with lots of servers. // If we run out of file descriptors, we might run into problems accessing diff --git a/setup/jetstream/nats.go b/setup/jetstream/nats.go index 48683789bd..06a58d5422 100644 --- a/setup/jetstream/nats.go +++ b/setup/jetstream/nats.go @@ -56,7 +56,9 @@ func (s *NATSInstance) Prepare(process *process.ProcessContext, cfg *config.JetS if err != nil { panic(err) } - s.SetLogger(NewLogAdapter(), opts.Debug, opts.Trace) + if !cfg.NoLog { + s.SetLogger(NewLogAdapter(), opts.Debug, opts.Trace) + } go func() { process.ComponentStarted() s.Start() diff --git a/setup/monolith.go b/setup/monolith.go index 174eba680f..51e40070b6 100644 --- a/setup/monolith.go +++ b/setup/monolith.go @@ -21,13 +21,16 @@ import ( "github.com/matrix-org/dendrite/federationapi" federationAPI "github.com/matrix-org/dendrite/federationapi/api" "github.com/matrix-org/dendrite/internal/caching" + "github.com/matrix-org/dendrite/internal/httputil" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/internal/transactions" "github.com/matrix-org/dendrite/mediaapi" "github.com/matrix-org/dendrite/relayapi" relayAPI "github.com/matrix-org/dendrite/relayapi/api" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" - "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/dendrite/setup/jetstream" + "github.com/matrix-org/dendrite/setup/process" "github.com/matrix-org/dendrite/syncapi" userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" @@ -53,23 +56,31 @@ type Monolith struct { } // AddAllPublicRoutes attaches all public paths to the given router -func (m *Monolith) AddAllPublicRoutes(base *base.BaseDendrite, caches *caching.Caches) { +func (m *Monolith) AddAllPublicRoutes( + processCtx *process.ProcessContext, + cfg *config.Dendrite, + routers httputil.Routers, + cm sqlutil.Connections, + natsInstance *jetstream.NATSInstance, + caches *caching.Caches, + enableMetrics bool, +) { userDirectoryProvider := m.ExtUserDirectoryProvider if userDirectoryProvider == nil { userDirectoryProvider = m.UserAPI } clientapi.AddPublicRoutes( - base, m.FedClient, m.RoomserverAPI, m.AppserviceAPI, transactions.New(), + processCtx, routers, cfg, natsInstance, m.FedClient, m.RoomserverAPI, m.AppserviceAPI, transactions.New(), m.FederationAPI, m.UserAPI, userDirectoryProvider, - m.ExtPublicRoomsProvider, + m.ExtPublicRoomsProvider, enableMetrics, ) federationapi.AddPublicRoutes( - base, m.UserAPI, m.FedClient, m.KeyRing, m.RoomserverAPI, m.FederationAPI, nil, + processCtx, routers, cfg, natsInstance, m.UserAPI, m.FedClient, m.KeyRing, m.RoomserverAPI, m.FederationAPI, nil, enableMetrics, ) - mediaapi.AddPublicRoutes(base, m.UserAPI, m.Client) - syncapi.AddPublicRoutes(base, m.UserAPI, m.RoomserverAPI, caches) + mediaapi.AddPublicRoutes(routers.Media, cm, cfg, m.UserAPI, m.Client) + syncapi.AddPublicRoutes(processCtx, routers, cfg, cm, natsInstance, m.UserAPI, m.RoomserverAPI, caches, enableMetrics) if m.RelayAPI != nil { - relayapi.AddPublicRoutes(base, m.KeyRing, m.RelayAPI) + relayapi.AddPublicRoutes(routers, cfg, m.KeyRing, m.RelayAPI) } } diff --git a/setup/mscs/msc2836/msc2836.go b/setup/mscs/msc2836/msc2836.go index 7c1e0fc61f..92248bc101 100644 --- a/setup/mscs/msc2836/msc2836.go +++ b/setup/mscs/msc2836/msc2836.go @@ -31,8 +31,9 @@ import ( fs "github.com/matrix-org/dendrite/federationapi/api" "github.com/matrix-org/dendrite/internal/hooks" "github.com/matrix-org/dendrite/internal/httputil" + "github.com/matrix-org/dendrite/internal/sqlutil" roomserver "github.com/matrix-org/dendrite/roomserver/api" - "github.com/matrix-org/dendrite/setup/base" + "github.com/matrix-org/dendrite/setup/config" userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" @@ -99,10 +100,10 @@ func toClientResponse(res *MSC2836EventRelationshipsResponse) *EventRelationship // Enable this MSC func Enable( - base *base.BaseDendrite, rsAPI roomserver.RoomserverInternalAPI, fsAPI fs.FederationInternalAPI, + cfg *config.Dendrite, cm sqlutil.Connections, routers httputil.Routers, rsAPI roomserver.RoomserverInternalAPI, fsAPI fs.FederationInternalAPI, userAPI userapi.UserInternalAPI, keyRing gomatrixserverlib.JSONVerifier, ) error { - db, err := NewDatabase(base.ConnectionManager, &base.Cfg.MSCs.Database) + db, err := NewDatabase(cm, &cfg.MSCs.Database) if err != nil { return fmt.Errorf("cannot enable MSC2836: %w", err) } @@ -125,14 +126,14 @@ func Enable( } }) - base.Routers.Client.Handle("/unstable/event_relationships", + routers.Client.Handle("/unstable/event_relationships", httputil.MakeAuthAPI("eventRelationships", userAPI, eventRelationshipHandler(db, rsAPI, fsAPI)), ).Methods(http.MethodPost, http.MethodOptions) - base.Routers.Federation.Handle("/unstable/event_relationships", httputil.MakeExternalAPI( + routers.Federation.Handle("/unstable/event_relationships", httputil.MakeExternalAPI( "msc2836_event_relationships", func(req *http.Request) util.JSONResponse { fedReq, errResp := gomatrixserverlib.VerifyHTTPRequest( - req, time.Now(), base.Cfg.Global.ServerName, base.Cfg.Global.IsLocalServerName, keyRing, + req, time.Now(), cfg.Global.ServerName, cfg.Global.IsLocalServerName, keyRing, ) if fedReq == nil { return errResp diff --git a/setup/mscs/msc2836/msc2836_test.go b/setup/mscs/msc2836/msc2836_test.go index 24e96f931f..bfcabef640 100644 --- a/setup/mscs/msc2836/msc2836_test.go +++ b/setup/mscs/msc2836/msc2836_test.go @@ -15,13 +15,13 @@ import ( "time" "github.com/gorilla/mux" + "github.com/matrix-org/dendrite/setup/process" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/dendrite/internal/hooks" "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/internal/sqlutil" roomserver "github.com/matrix-org/dendrite/roomserver/api" - "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/mscs/msc2836" userapi "github.com/matrix-org/dendrite/userapi/api" @@ -555,21 +555,18 @@ func injectEvents(t *testing.T, userAPI userapi.UserInternalAPI, rsAPI roomserve cfg.Global.ServerName = "localhost" cfg.MSCs.Database.ConnectionString = "file:msc2836_test.db" cfg.MSCs.MSCs = []string{"msc2836"} - cm := sqlutil.NewConnectionManager() - base := &base.BaseDendrite{ - Cfg: cfg, - Routers: httputil.NewRouters(), - ConnectionManager: cm, - } - err := msc2836.Enable(base, rsAPI, nil, userAPI, nil) + processCtx := process.NewProcessContext() + cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) + routers := httputil.NewRouters() + err := msc2836.Enable(cfg, cm, routers, rsAPI, nil, userAPI, nil) if err != nil { t.Fatalf("failed to enable MSC2836: %s", err) } for _, ev := range events { hooks.Run(hooks.KindNewEventPersisted, ev) } - return base.Routers.Client + return routers.Client } type fledglingEvent struct { diff --git a/setup/mscs/msc2946/msc2946.go b/setup/mscs/msc2946/msc2946.go index b4b93ff391..d33897b936 100644 --- a/setup/mscs/msc2946/msc2946.go +++ b/setup/mscs/msc2946/msc2946.go @@ -33,7 +33,7 @@ import ( "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/internal/httputil" roomserver "github.com/matrix-org/dendrite/roomserver/api" - "github.com/matrix-org/dendrite/setup/base" + "github.com/matrix-org/dendrite/setup/config" userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" @@ -54,17 +54,17 @@ type MSC2946ClientResponse struct { // Enable this MSC func Enable( - base *base.BaseDendrite, rsAPI roomserver.RoomserverInternalAPI, userAPI userapi.UserInternalAPI, + cfg *config.Dendrite, routers httputil.Routers, rsAPI roomserver.RoomserverInternalAPI, userAPI userapi.UserInternalAPI, fsAPI fs.FederationInternalAPI, keyRing gomatrixserverlib.JSONVerifier, cache caching.SpaceSummaryRoomsCache, ) error { - clientAPI := httputil.MakeAuthAPI("spaces", userAPI, spacesHandler(rsAPI, fsAPI, cache, base.Cfg.Global.ServerName), httputil.WithAllowGuests()) - base.Routers.Client.Handle("/v1/rooms/{roomID}/hierarchy", clientAPI).Methods(http.MethodGet, http.MethodOptions) - base.Routers.Client.Handle("/unstable/org.matrix.msc2946/rooms/{roomID}/hierarchy", clientAPI).Methods(http.MethodGet, http.MethodOptions) + clientAPI := httputil.MakeAuthAPI("spaces", userAPI, spacesHandler(rsAPI, fsAPI, cache, cfg.Global.ServerName), httputil.WithAllowGuests()) + routers.Client.Handle("/v1/rooms/{roomID}/hierarchy", clientAPI).Methods(http.MethodGet, http.MethodOptions) + routers.Client.Handle("/unstable/org.matrix.msc2946/rooms/{roomID}/hierarchy", clientAPI).Methods(http.MethodGet, http.MethodOptions) fedAPI := httputil.MakeExternalAPI( "msc2946_fed_spaces", func(req *http.Request) util.JSONResponse { fedReq, errResp := gomatrixserverlib.VerifyHTTPRequest( - req, time.Now(), base.Cfg.Global.ServerName, base.Cfg.Global.IsLocalServerName, keyRing, + req, time.Now(), cfg.Global.ServerName, cfg.Global.IsLocalServerName, keyRing, ) if fedReq == nil { return errResp @@ -75,11 +75,11 @@ func Enable( return util.ErrorResponse(err) } roomID := params["roomID"] - return federatedSpacesHandler(req.Context(), fedReq, roomID, cache, rsAPI, fsAPI, base.Cfg.Global.ServerName) + return federatedSpacesHandler(req.Context(), fedReq, roomID, cache, rsAPI, fsAPI, cfg.Global.ServerName) }, ) - base.Routers.Federation.Handle("/unstable/org.matrix.msc2946/hierarchy/{roomID}", fedAPI).Methods(http.MethodGet) - base.Routers.Federation.Handle("/v1/hierarchy/{roomID}", fedAPI).Methods(http.MethodGet) + routers.Federation.Handle("/unstable/org.matrix.msc2946/hierarchy/{roomID}", fedAPI).Methods(http.MethodGet) + routers.Federation.Handle("/v1/hierarchy/{roomID}", fedAPI).Methods(http.MethodGet) return nil } diff --git a/setup/mscs/mscs.go b/setup/mscs/mscs.go index b58c800b05..9cd5eed1cd 100644 --- a/setup/mscs/mscs.go +++ b/setup/mscs/mscs.go @@ -20,30 +20,32 @@ import ( "fmt" "github.com/matrix-org/dendrite/internal/caching" + "github.com/matrix-org/dendrite/internal/httputil" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/setup" - "github.com/matrix-org/dendrite/setup/base" + "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/mscs/msc2836" "github.com/matrix-org/dendrite/setup/mscs/msc2946" "github.com/matrix-org/util" ) // Enable MSCs - returns an error on unknown MSCs -func Enable(base *base.BaseDendrite, monolith *setup.Monolith, caches *caching.Caches) error { - for _, msc := range base.Cfg.MSCs.MSCs { +func Enable(cfg *config.Dendrite, cm sqlutil.Connections, routers httputil.Routers, monolith *setup.Monolith, caches *caching.Caches) error { + for _, msc := range cfg.MSCs.MSCs { util.GetLogger(context.Background()).WithField("msc", msc).Info("Enabling MSC") - if err := EnableMSC(base, monolith, msc, caches); err != nil { + if err := EnableMSC(cfg, cm, routers, monolith, msc, caches); err != nil { return err } } return nil } -func EnableMSC(base *base.BaseDendrite, monolith *setup.Monolith, msc string, caches *caching.Caches) error { +func EnableMSC(cfg *config.Dendrite, cm sqlutil.Connections, routers httputil.Routers, monolith *setup.Monolith, msc string, caches *caching.Caches) error { switch msc { case "msc2836": - return msc2836.Enable(base, monolith.RoomserverAPI, monolith.FederationAPI, monolith.UserAPI, monolith.KeyRing) + return msc2836.Enable(cfg, cm, routers, monolith.RoomserverAPI, monolith.FederationAPI, monolith.UserAPI, monolith.KeyRing) case "msc2946": - return msc2946.Enable(base, monolith.RoomserverAPI, monolith.UserAPI, monolith.FederationAPI, monolith.KeyRing, caches) + return msc2946.Enable(cfg, routers, monolith.RoomserverAPI, monolith.UserAPI, monolith.FederationAPI, monolith.KeyRing, caches) case "msc2444": // enabled inside federationapi case "msc2753": // enabled inside clientapi default: diff --git a/syncapi/storage/storage_test.go b/syncapi/storage/storage_test.go index 9f0064907b..e81a341f19 100644 --- a/syncapi/storage/storage_test.go +++ b/syncapi/storage/storage_test.go @@ -22,7 +22,7 @@ var ctx = context.Background() func MustCreateDatabase(t *testing.T, dbType test.DBType) (storage.Database, func()) { connStr, close := test.PrepareDBConnectionString(t, dbType) - cm := sqlutil.NewConnectionManager() + cm := sqlutil.NewConnectionManager(nil, config.DatabaseOptions{}) db, err := storage.NewSyncServerDatasource(context.Background(), cm, &config.DatabaseOptions{ ConnectionString: config.DataSource(connStr), }) diff --git a/syncapi/syncapi.go b/syncapi/syncapi.go index e0cc8462e2..9a27b954a4 100644 --- a/syncapi/syncapi.go +++ b/syncapi/syncapi.go @@ -18,12 +18,15 @@ import ( "context" "github.com/matrix-org/dendrite/internal/fulltext" + "github.com/matrix-org/dendrite/internal/httputil" + "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/dendrite/setup/process" "github.com/sirupsen/logrus" "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/roomserver/api" - "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/jetstream" userapi "github.com/matrix-org/dendrite/userapi/api" @@ -39,16 +42,19 @@ import ( // AddPublicRoutes sets up and registers HTTP handlers for the SyncAPI // component. func AddPublicRoutes( - base *base.BaseDendrite, + processContext *process.ProcessContext, + routers httputil.Routers, + dendriteCfg *config.Dendrite, + cm sqlutil.Connections, + natsInstance *jetstream.NATSInstance, userAPI userapi.SyncUserAPI, rsAPI api.SyncRoomserverAPI, caches caching.LazyLoadCache, + enableMetrics bool, ) { - cfg := &base.Cfg.SyncAPI + js, natsClient := natsInstance.Prepare(processContext, &dendriteCfg.Global.JetStream) - js, natsClient := base.NATS.Prepare(base.ProcessContext, &cfg.Matrix.JetStream) - - syncDB, err := storage.NewSyncServerDatasource(base.Context(), base.ConnectionManager, &cfg.Database) + syncDB, err := storage.NewSyncServerDatasource(processContext.Context(), cm, &dendriteCfg.SyncAPI.Database) if err != nil { logrus.WithError(err).Panicf("failed to connect to sync db") } @@ -62,32 +68,32 @@ func AddPublicRoutes( } var fts *fulltext.Search - if cfg.Fulltext.Enabled { - fts, err = fulltext.New(base.ProcessContext.Context(), cfg.Fulltext) + if dendriteCfg.SyncAPI.Fulltext.Enabled { + fts, err = fulltext.New(processContext.Context(), dendriteCfg.SyncAPI.Fulltext) if err != nil { logrus.WithError(err).Panicf("failed to create full text") } - base.ProcessContext.ComponentStarted() + processContext.ComponentStarted() } federationPresenceProducer := &producers.FederationAPIPresenceProducer{ - Topic: cfg.Matrix.JetStream.Prefixed(jetstream.OutputPresenceEvent), + Topic: dendriteCfg.Global.JetStream.Prefixed(jetstream.OutputPresenceEvent), JetStream: js, } presenceConsumer := consumers.NewPresenceConsumer( - base.ProcessContext, cfg, js, natsClient, syncDB, + processContext, &dendriteCfg.SyncAPI, js, natsClient, syncDB, notifier, streams.PresenceStreamProvider, userAPI, ) - requestPool := sync.NewRequestPool(syncDB, cfg, userAPI, rsAPI, streams, notifier, federationPresenceProducer, presenceConsumer, base.EnableMetrics) + requestPool := sync.NewRequestPool(syncDB, &dendriteCfg.SyncAPI, userAPI, rsAPI, streams, notifier, federationPresenceProducer, presenceConsumer, enableMetrics) if err = presenceConsumer.Start(); err != nil { logrus.WithError(err).Panicf("failed to start presence consumer") } keyChangeConsumer := consumers.NewOutputKeyChangeEventConsumer( - base.ProcessContext, cfg, cfg.Matrix.JetStream.Prefixed(jetstream.OutputKeyChangeEvent), + processContext, &dendriteCfg.SyncAPI, dendriteCfg.Global.JetStream.Prefixed(jetstream.OutputKeyChangeEvent), js, rsAPI, syncDB, notifier, streams.DeviceListStreamProvider, ) @@ -96,7 +102,7 @@ func AddPublicRoutes( } roomConsumer := consumers.NewOutputRoomEventConsumer( - base.ProcessContext, cfg, js, syncDB, notifier, streams.PDUStreamProvider, + processContext, &dendriteCfg.SyncAPI, js, syncDB, notifier, streams.PDUStreamProvider, streams.InviteStreamProvider, rsAPI, fts, ) if err = roomConsumer.Start(); err != nil { @@ -104,7 +110,7 @@ func AddPublicRoutes( } clientConsumer := consumers.NewOutputClientDataConsumer( - base.ProcessContext, cfg, js, natsClient, syncDB, notifier, + processContext, &dendriteCfg.SyncAPI, js, natsClient, syncDB, notifier, streams.AccountDataStreamProvider, fts, ) if err = clientConsumer.Start(); err != nil { @@ -112,35 +118,35 @@ func AddPublicRoutes( } notificationConsumer := consumers.NewOutputNotificationDataConsumer( - base.ProcessContext, cfg, js, syncDB, notifier, streams.NotificationDataStreamProvider, + processContext, &dendriteCfg.SyncAPI, js, syncDB, notifier, streams.NotificationDataStreamProvider, ) if err = notificationConsumer.Start(); err != nil { logrus.WithError(err).Panicf("failed to start notification data consumer") } typingConsumer := consumers.NewOutputTypingEventConsumer( - base.ProcessContext, cfg, js, eduCache, notifier, streams.TypingStreamProvider, + processContext, &dendriteCfg.SyncAPI, js, eduCache, notifier, streams.TypingStreamProvider, ) if err = typingConsumer.Start(); err != nil { logrus.WithError(err).Panicf("failed to start typing consumer") } sendToDeviceConsumer := consumers.NewOutputSendToDeviceEventConsumer( - base.ProcessContext, cfg, js, syncDB, userAPI, notifier, streams.SendToDeviceStreamProvider, + processContext, &dendriteCfg.SyncAPI, js, syncDB, userAPI, notifier, streams.SendToDeviceStreamProvider, ) if err = sendToDeviceConsumer.Start(); err != nil { logrus.WithError(err).Panicf("failed to start send-to-device consumer") } receiptConsumer := consumers.NewOutputReceiptEventConsumer( - base.ProcessContext, cfg, js, syncDB, notifier, streams.ReceiptStreamProvider, + processContext, &dendriteCfg.SyncAPI, js, syncDB, notifier, streams.ReceiptStreamProvider, ) if err = receiptConsumer.Start(); err != nil { logrus.WithError(err).Panicf("failed to start receipts consumer") } routing.Setup( - base.Routers.Client, requestPool, syncDB, userAPI, - rsAPI, cfg, caches, fts, + routers.Client, requestPool, syncDB, userAPI, + rsAPI, &dendriteCfg.SyncAPI, caches, fts, ) } diff --git a/syncapi/syncapi_test.go b/syncapi/syncapi_test.go index 13a078659c..584782afb2 100644 --- a/syncapi/syncapi_test.go +++ b/syncapi/syncapi_test.go @@ -11,6 +11,9 @@ import ( "time" "github.com/matrix-org/dendrite/internal/caching" + "github.com/matrix-org/dendrite/internal/httputil" + "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/gomatrixserverlib" "github.com/nats-io/nats.go" "github.com/tidwall/gjson" @@ -22,7 +25,6 @@ import ( "github.com/matrix-org/dendrite/roomserver" "github.com/matrix-org/dendrite/roomserver/api" rsapi "github.com/matrix-org/dendrite/roomserver/api" - "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/jetstream" "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/dendrite/test" @@ -114,14 +116,17 @@ func testSyncAccessTokens(t *testing.T, dbType test.DBType) { AccountType: userapi.AccountTypeUser, } - base, close := testrig.CreateBaseDendrite(t, dbType) - caches := caching.NewRistrettoCache(base.Cfg.Global.Cache.EstimatedMaxSize, base.Cfg.Global.Cache.MaxAge, caching.DisableMetrics) + cfg, processCtx, close := testrig.CreateConfig(t, dbType) + routers := httputil.NewRouters() + cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) + caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) + natsInstance := jetstream.NATSInstance{} defer close() - jsctx, _ := base.NATS.Prepare(base.ProcessContext, &base.Cfg.Global.JetStream) - defer jetstream.DeleteAllStreams(jsctx, &base.Cfg.Global.JetStream) - msgs := toNATSMsgs(t, base, room.Events()...) - AddPublicRoutes(base, &syncUserAPI{accounts: []userapi.Device{alice}}, &syncRoomserverAPI{rooms: []*test.Room{room}}, caches) + jsctx, _ := natsInstance.Prepare(processCtx, &cfg.Global.JetStream) + defer jetstream.DeleteAllStreams(jsctx, &cfg.Global.JetStream) + msgs := toNATSMsgs(t, cfg, room.Events()...) + AddPublicRoutes(processCtx, routers, cfg, cm, &natsInstance, &syncUserAPI{accounts: []userapi.Device{alice}}, &syncRoomserverAPI{rooms: []*test.Room{room}}, caches, caching.DisableMetrics) testrig.MustPublishMsgs(t, jsctx, msgs...) testCases := []struct { @@ -156,7 +161,7 @@ func testSyncAccessTokens(t *testing.T, dbType test.DBType) { }, } - syncUntil(t, base, alice.AccessToken, false, func(syncBody string) bool { + syncUntil(t, routers, alice.AccessToken, false, func(syncBody string) bool { // wait for the last sent eventID to come down sync path := fmt.Sprintf(`rooms.join.%s.timeline.events.#(event_id=="%s")`, room.ID, room.Events()[len(room.Events())-1].EventID()) return gjson.Get(syncBody, path).Exists() @@ -164,7 +169,7 @@ func testSyncAccessTokens(t *testing.T, dbType test.DBType) { for _, tc := range testCases { w := httptest.NewRecorder() - base.Routers.Client.ServeHTTP(w, tc.req) + routers.Client.ServeHTTP(w, tc.req) if w.Code != tc.wantCode { t.Fatalf("%s: got HTTP %d want %d", tc.name, w.Code, tc.wantCode) } @@ -207,26 +212,29 @@ func testSyncAPICreateRoomSyncEarly(t *testing.T, dbType test.DBType) { AccountType: userapi.AccountTypeUser, } - base, close := testrig.CreateBaseDendrite(t, dbType) + cfg, processCtx, close := testrig.CreateConfig(t, dbType) + routers := httputil.NewRouters() + cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) + caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) defer close() + natsInstance := jetstream.NATSInstance{} - jsctx, _ := base.NATS.Prepare(base.ProcessContext, &base.Cfg.Global.JetStream) - defer jetstream.DeleteAllStreams(jsctx, &base.Cfg.Global.JetStream) + jsctx, _ := natsInstance.Prepare(processCtx, &cfg.Global.JetStream) + defer jetstream.DeleteAllStreams(jsctx, &cfg.Global.JetStream) // order is: // m.room.create // m.room.member // m.room.power_levels // m.room.join_rules // m.room.history_visibility - msgs := toNATSMsgs(t, base, room.Events()...) + msgs := toNATSMsgs(t, cfg, room.Events()...) sinceTokens := make([]string, len(msgs)) - caches := caching.NewRistrettoCache(base.Cfg.Global.Cache.EstimatedMaxSize, base.Cfg.Global.Cache.MaxAge, caching.DisableMetrics) - AddPublicRoutes(base, &syncUserAPI{accounts: []userapi.Device{alice}}, &syncRoomserverAPI{rooms: []*test.Room{room}}, caches) + AddPublicRoutes(processCtx, routers, cfg, cm, &natsInstance, &syncUserAPI{accounts: []userapi.Device{alice}}, &syncRoomserverAPI{rooms: []*test.Room{room}}, caches, caching.DisableMetrics) for i, msg := range msgs { testrig.MustPublishMsgs(t, jsctx, msg) time.Sleep(100 * time.Millisecond) w := httptest.NewRecorder() - base.Routers.Client.ServeHTTP(w, test.NewRequest(t, "GET", "/_matrix/client/v3/sync", test.WithQueryParams(map[string]string{ + routers.Client.ServeHTTP(w, test.NewRequest(t, "GET", "/_matrix/client/v3/sync", test.WithQueryParams(map[string]string{ "access_token": alice.AccessToken, "timeout": "0", }))) @@ -256,7 +264,7 @@ func testSyncAPICreateRoomSyncEarly(t *testing.T, dbType test.DBType) { t.Logf("waited for events to be consumed; syncing with %v", sinceTokens) for i, since := range sinceTokens { w := httptest.NewRecorder() - base.Routers.Client.ServeHTTP(w, test.NewRequest(t, "GET", "/_matrix/client/v3/sync", test.WithQueryParams(map[string]string{ + routers.Client.ServeHTTP(w, test.NewRequest(t, "GET", "/_matrix/client/v3/sync", test.WithQueryParams(map[string]string{ "access_token": alice.AccessToken, "timeout": "0", "since": since, @@ -298,17 +306,20 @@ func testSyncAPIUpdatePresenceImmediately(t *testing.T, dbType test.DBType) { AccountType: userapi.AccountTypeUser, } - base, close := testrig.CreateBaseDendrite(t, dbType) - base.Cfg.Global.Presence.EnableOutbound = true - base.Cfg.Global.Presence.EnableInbound = true + cfg, processCtx, close := testrig.CreateConfig(t, dbType) + routers := httputil.NewRouters() + cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) + caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) + cfg.Global.Presence.EnableOutbound = true + cfg.Global.Presence.EnableInbound = true defer close() + natsInstance := jetstream.NATSInstance{} - jsctx, _ := base.NATS.Prepare(base.ProcessContext, &base.Cfg.Global.JetStream) - defer jetstream.DeleteAllStreams(jsctx, &base.Cfg.Global.JetStream) - caches := caching.NewRistrettoCache(base.Cfg.Global.Cache.EstimatedMaxSize, base.Cfg.Global.Cache.MaxAge, caching.DisableMetrics) - AddPublicRoutes(base, &syncUserAPI{accounts: []userapi.Device{alice}}, &syncRoomserverAPI{}, caches) + jsctx, _ := natsInstance.Prepare(processCtx, &cfg.Global.JetStream) + defer jetstream.DeleteAllStreams(jsctx, &cfg.Global.JetStream) + AddPublicRoutes(processCtx, routers, cfg, cm, &natsInstance, &syncUserAPI{accounts: []userapi.Device{alice}}, &syncRoomserverAPI{}, caches, caching.DisableMetrics) w := httptest.NewRecorder() - base.Routers.Client.ServeHTTP(w, test.NewRequest(t, "GET", "/_matrix/client/v3/sync", test.WithQueryParams(map[string]string{ + routers.Client.ServeHTTP(w, test.NewRequest(t, "GET", "/_matrix/client/v3/sync", test.WithQueryParams(map[string]string{ "access_token": alice.AccessToken, "timeout": "0", "set_presence": "online", @@ -414,17 +425,20 @@ func testHistoryVisibility(t *testing.T, dbType test.DBType) { userType = "real user" } - base, close := testrig.CreateBaseDendrite(t, dbType) + cfg, processCtx, close := testrig.CreateConfig(t, dbType) + routers := httputil.NewRouters() + cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) + caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) defer close() + natsInstance := jetstream.NATSInstance{} - jsctx, _ := base.NATS.Prepare(base.ProcessContext, &base.Cfg.Global.JetStream) - defer jetstream.DeleteAllStreams(jsctx, &base.Cfg.Global.JetStream) + jsctx, _ := natsInstance.Prepare(processCtx, &cfg.Global.JetStream) + defer jetstream.DeleteAllStreams(jsctx, &cfg.Global.JetStream) // Use the actual internal roomserver API - caches := caching.NewRistrettoCache(base.Cfg.Global.Cache.EstimatedMaxSize, base.Cfg.Global.Cache.MaxAge, caching.DisableMetrics) - rsAPI := roomserver.NewInternalAPI(base, caches) + rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - AddPublicRoutes(base, &syncUserAPI{accounts: []userapi.Device{aliceDev, bobDev}}, rsAPI, caches) + AddPublicRoutes(processCtx, routers, cfg, cm, &natsInstance, &syncUserAPI{accounts: []userapi.Device{aliceDev, bobDev}}, rsAPI, caches, caching.DisableMetrics) for _, tc := range testCases { testname := fmt.Sprintf("%s - %s", tc.historyVisibility, userType) @@ -439,7 +453,7 @@ func testHistoryVisibility(t *testing.T, dbType test.DBType) { if err := api.SendEvents(ctx, rsAPI, api.KindNew, eventsToSend, "test", "test", "test", nil, false); err != nil { t.Fatalf("failed to send events: %v", err) } - syncUntil(t, base, aliceDev.AccessToken, false, + syncUntil(t, routers, aliceDev.AccessToken, false, func(syncBody string) bool { path := fmt.Sprintf(`rooms.join.%s.timeline.events.#(content.body=="%s")`, room.ID, beforeJoinBody) return gjson.Get(syncBody, path).Exists() @@ -448,7 +462,7 @@ func testHistoryVisibility(t *testing.T, dbType test.DBType) { // There is only one event, we expect only to be able to see this, if the room is world_readable w := httptest.NewRecorder() - base.Routers.Client.ServeHTTP(w, test.NewRequest(t, "GET", fmt.Sprintf("/_matrix/client/v3/rooms/%s/messages", room.ID), test.WithQueryParams(map[string]string{ + routers.Client.ServeHTTP(w, test.NewRequest(t, "GET", fmt.Sprintf("/_matrix/client/v3/rooms/%s/messages", room.ID), test.WithQueryParams(map[string]string{ "access_token": bobDev.AccessToken, "dir": "b", "filter": `{"lazy_load_members":true}`, // check that lazy loading doesn't break history visibility @@ -479,7 +493,7 @@ func testHistoryVisibility(t *testing.T, dbType test.DBType) { if err := api.SendEvents(ctx, rsAPI, api.KindNew, eventsToSend, "test", "test", "test", nil, false); err != nil { t.Fatalf("failed to send events: %v", err) } - syncUntil(t, base, aliceDev.AccessToken, false, + syncUntil(t, routers, aliceDev.AccessToken, false, func(syncBody string) bool { path := fmt.Sprintf(`rooms.join.%s.timeline.events.#(content.body=="%s")`, room.ID, afterJoinBody) return gjson.Get(syncBody, path).Exists() @@ -488,7 +502,7 @@ func testHistoryVisibility(t *testing.T, dbType test.DBType) { // Verify the messages after/before invite are visible or not w = httptest.NewRecorder() - base.Routers.Client.ServeHTTP(w, test.NewRequest(t, "GET", fmt.Sprintf("/_matrix/client/v3/rooms/%s/messages", room.ID), test.WithQueryParams(map[string]string{ + routers.Client.ServeHTTP(w, test.NewRequest(t, "GET", fmt.Sprintf("/_matrix/client/v3/rooms/%s/messages", room.ID), test.WithQueryParams(map[string]string{ "access_token": bobDev.AccessToken, "dir": "b", }))) @@ -714,18 +728,20 @@ func TestGetMembership(t *testing.T) { test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { - base, close := testrig.CreateBaseDendrite(t, dbType) + cfg, processCtx, close := testrig.CreateConfig(t, dbType) + routers := httputil.NewRouters() + cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) + caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) defer close() - - jsctx, _ := base.NATS.Prepare(base.ProcessContext, &base.Cfg.Global.JetStream) - defer jetstream.DeleteAllStreams(jsctx, &base.Cfg.Global.JetStream) + natsInstance := jetstream.NATSInstance{} + jsctx, _ := natsInstance.Prepare(processCtx, &cfg.Global.JetStream) + defer jetstream.DeleteAllStreams(jsctx, &cfg.Global.JetStream) // Use an actual roomserver for this - caches := caching.NewRistrettoCache(base.Cfg.Global.Cache.EstimatedMaxSize, base.Cfg.Global.Cache.MaxAge, caching.DisableMetrics) - rsAPI := roomserver.NewInternalAPI(base, caches) + rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - AddPublicRoutes(base, &syncUserAPI{accounts: []userapi.Device{aliceDev, bobDev}}, rsAPI, caches) + AddPublicRoutes(processCtx, routers, cfg, cm, &natsInstance, &syncUserAPI{accounts: []userapi.Device{aliceDev, bobDev}}, rsAPI, caches, caching.DisableMetrics) for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { @@ -745,7 +761,7 @@ func TestGetMembership(t *testing.T) { if tc.useSleep { time.Sleep(time.Millisecond * 100) } else { - syncUntil(t, base, aliceDev.AccessToken, false, func(syncBody string) bool { + syncUntil(t, routers, aliceDev.AccessToken, false, func(syncBody string) bool { // wait for the last sent eventID to come down sync path := fmt.Sprintf(`rooms.join.%s.timeline.events.#(event_id=="%s")`, room.ID, room.Events()[len(room.Events())-1].EventID()) return gjson.Get(syncBody, path).Exists() @@ -753,7 +769,7 @@ func TestGetMembership(t *testing.T) { } w := httptest.NewRecorder() - base.Routers.Client.ServeHTTP(w, tc.request(t, room)) + routers.Client.ServeHTTP(w, tc.request(t, room)) if w.Code != 200 && tc.wantOK { t.Logf("%s", w.Body.String()) t.Fatalf("got HTTP %d want %d", w.Code, 200) @@ -786,16 +802,19 @@ func testSendToDevice(t *testing.T, dbType test.DBType) { AccountType: userapi.AccountTypeUser, } - base, baseClose := testrig.CreateBaseDendrite(t, dbType) - defer baseClose() + cfg, processCtx, close := testrig.CreateConfig(t, dbType) + routers := httputil.NewRouters() + cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) + caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) + defer close() + natsInstance := jetstream.NATSInstance{} - jsctx, _ := base.NATS.Prepare(base.ProcessContext, &base.Cfg.Global.JetStream) - defer jetstream.DeleteAllStreams(jsctx, &base.Cfg.Global.JetStream) - caches := caching.NewRistrettoCache(base.Cfg.Global.Cache.EstimatedMaxSize, base.Cfg.Global.Cache.MaxAge, caching.DisableMetrics) - AddPublicRoutes(base, &syncUserAPI{accounts: []userapi.Device{alice}}, &syncRoomserverAPI{}, caches) + jsctx, _ := natsInstance.Prepare(processCtx, &cfg.Global.JetStream) + defer jetstream.DeleteAllStreams(jsctx, &cfg.Global.JetStream) + AddPublicRoutes(processCtx, routers, cfg, cm, &natsInstance, &syncUserAPI{accounts: []userapi.Device{alice}}, &syncRoomserverAPI{}, caches, caching.DisableMetrics) producer := producers.SyncAPIProducer{ - TopicSendToDeviceEvent: base.Cfg.Global.JetStream.Prefixed(jetstream.OutputSendToDeviceEvent), + TopicSendToDeviceEvent: cfg.Global.JetStream.Prefixed(jetstream.OutputSendToDeviceEvent), JetStream: jsctx, } @@ -881,7 +900,7 @@ func testSendToDevice(t *testing.T, dbType test.DBType) { } } - syncUntil(t, base, alice.AccessToken, + syncUntil(t, routers, alice.AccessToken, len(tc.want) == 0, func(body string) bool { return gjson.Get(body, fmt.Sprintf(`to_device.events.#(content.dummy=="message %d")`, msgCounter)).Exists() @@ -890,7 +909,7 @@ func testSendToDevice(t *testing.T, dbType test.DBType) { // Execute a /sync request, recording the response w := httptest.NewRecorder() - base.Routers.Client.ServeHTTP(w, test.NewRequest(t, "GET", "/_matrix/client/v3/sync", test.WithQueryParams(map[string]string{ + routers.Client.ServeHTTP(w, test.NewRequest(t, "GET", "/_matrix/client/v3/sync", test.WithQueryParams(map[string]string{ "access_token": alice.AccessToken, "since": tc.since, }))) @@ -1004,15 +1023,18 @@ func testContext(t *testing.T, dbType test.DBType) { AccountType: userapi.AccountTypeUser, } - base, baseClose := testrig.CreateBaseDendrite(t, dbType) - defer baseClose() + cfg, processCtx, close := testrig.CreateConfig(t, dbType) + routers := httputil.NewRouters() + cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) + caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) + defer close() // Use an actual roomserver for this - caches := caching.NewRistrettoCache(base.Cfg.Global.Cache.EstimatedMaxSize, base.Cfg.Global.Cache.MaxAge, caching.DisableMetrics) - rsAPI := roomserver.NewInternalAPI(base, caches) + natsInstance := jetstream.NATSInstance{} + rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - AddPublicRoutes(base, &syncUserAPI{accounts: []userapi.Device{alice}}, rsAPI, caches) + AddPublicRoutes(processCtx, routers, cfg, cm, &natsInstance, &syncUserAPI{accounts: []userapi.Device{alice}}, rsAPI, caches, caching.DisableMetrics) room := test.NewRoom(t, user) @@ -1025,10 +1047,10 @@ func testContext(t *testing.T, dbType test.DBType) { t.Fatalf("failed to send events: %v", err) } - jsctx, _ := base.NATS.Prepare(base.ProcessContext, &base.Cfg.Global.JetStream) - defer jetstream.DeleteAllStreams(jsctx, &base.Cfg.Global.JetStream) + jsctx, _ := natsInstance.Prepare(processCtx, &cfg.Global.JetStream) + defer jetstream.DeleteAllStreams(jsctx, &cfg.Global.JetStream) - syncUntil(t, base, alice.AccessToken, false, func(syncBody string) bool { + syncUntil(t, routers, alice.AccessToken, false, func(syncBody string) bool { // wait for the last sent eventID to come down sync path := fmt.Sprintf(`rooms.join.%s.timeline.events.#(event_id=="%s")`, room.ID, thirdMsg.EventID()) return gjson.Get(syncBody, path).Exists() @@ -1055,7 +1077,7 @@ func testContext(t *testing.T, dbType test.DBType) { params[k] = v } } - base.Routers.Client.ServeHTTP(w, test.NewRequest(t, "GET", requestPath, test.WithQueryParams(params))) + routers.Client.ServeHTTP(w, test.NewRequest(t, "GET", requestPath, test.WithQueryParams(params))) if tc.wantError && w.Code == 200 { t.Fatalf("Expected an error, but got none") @@ -1143,9 +1165,10 @@ func TestUpdateRelations(t *testing.T) { room := test.NewRoom(t, alice) test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { - base, shutdownBase := testrig.CreateBaseDendrite(t, dbType) - t.Cleanup(shutdownBase) - db, err := storage.NewSyncServerDatasource(base.Context(), base.ConnectionManager, &base.Cfg.SyncAPI.Database) + cfg, processCtx, close := testrig.CreateConfig(t, dbType) + cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) + t.Cleanup(close) + db, err := storage.NewSyncServerDatasource(processCtx.Context(), cm, &cfg.SyncAPI.Database) if err != nil { t.Fatal(err) } @@ -1167,10 +1190,11 @@ func TestUpdateRelations(t *testing.T) { } func syncUntil(t *testing.T, - base *base.BaseDendrite, accessToken string, + routers httputil.Routers, accessToken string, skip bool, checkFunc func(syncBody string) bool, ) { + t.Helper() if checkFunc == nil { t.Fatalf("No checkFunc defined") } @@ -1184,7 +1208,7 @@ func syncUntil(t *testing.T, go func() { for { w := httptest.NewRecorder() - base.Routers.Client.ServeHTTP(w, test.NewRequest(t, "GET", "/_matrix/client/v3/sync", test.WithQueryParams(map[string]string{ + routers.Client.ServeHTTP(w, test.NewRequest(t, "GET", "/_matrix/client/v3/sync", test.WithQueryParams(map[string]string{ "access_token": accessToken, "timeout": "1000", }))) @@ -1202,14 +1226,14 @@ func syncUntil(t *testing.T, } } -func toNATSMsgs(t *testing.T, base *base.BaseDendrite, input ...*gomatrixserverlib.HeaderedEvent) []*nats.Msg { +func toNATSMsgs(t *testing.T, cfg *config.Dendrite, input ...*gomatrixserverlib.HeaderedEvent) []*nats.Msg { result := make([]*nats.Msg, len(input)) for i, ev := range input { var addsStateIDs []string if ev.StateKey() != nil { addsStateIDs = append(addsStateIDs, ev.EventID()) } - result[i] = testrig.NewOutputEventMsg(t, base, ev.RoomID(), api.OutputEvent{ + result[i] = testrig.NewOutputEventMsg(t, cfg, ev.RoomID(), api.OutputEvent{ Type: rsapi.OutputTypeNewRoomEvent, NewRoomEvent: &rsapi.OutputNewRoomEvent{ Event: ev, diff --git a/test/testrig/base.go b/test/testrig/base.go index bb8fded217..fd85578ee3 100644 --- a/test/testrig/base.go +++ b/test/testrig/base.go @@ -19,13 +19,12 @@ import ( "path/filepath" "testing" - "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/dendrite/setup/process" "github.com/matrix-org/dendrite/test" - "github.com/nats-io/nats.go" ) -func CreateBaseDendrite(t *testing.T, dbType test.DBType) (*base.BaseDendrite, func()) { +func CreateConfig(t *testing.T, dbType test.DBType) (*config.Dendrite, *process.ProcessContext, func()) { var cfg config.Dendrite cfg.Defaults(config.DefaultOpts{ Generate: false, @@ -33,6 +32,7 @@ func CreateBaseDendrite(t *testing.T, dbType test.DBType) (*base.BaseDendrite, f }) cfg.Global.JetStream.InMemory = true cfg.FederationAPI.KeyPerspectives = nil + ctx := process.NewProcessContext() switch dbType { case test.DBTypePostgres: cfg.Global.Defaults(config.DefaultOpts{ // autogen a signing key @@ -51,18 +51,18 @@ func CreateBaseDendrite(t *testing.T, dbType test.DBType) (*base.BaseDendrite, f // use a distinct prefix else concurrent postgres/sqlite runs will clash since NATS will use // the file system event with InMemory=true :( cfg.Global.JetStream.TopicPrefix = fmt.Sprintf("Test_%d_", dbType) - connStr, close := test.PrepareDBConnectionString(t, dbType) + + connStr, closeDb := test.PrepareDBConnectionString(t, dbType) cfg.Global.DatabaseOptions = config.DatabaseOptions{ ConnectionString: config.DataSource(connStr), MaxOpenConnections: 10, MaxIdleConnections: 2, ConnMaxLifetimeSeconds: 60, } - base := base.NewBaseDendrite(&cfg, base.DisableMetrics) - return base, func() { - base.ShutdownDendrite() - base.WaitForShutdown() - close() + return &cfg, ctx, func() { + ctx.ShutdownDendrite() + ctx.WaitForShutdown() + closeDb() } case test.DBTypeSQLite: cfg.Defaults(config.DefaultOpts{ @@ -86,30 +86,13 @@ func CreateBaseDendrite(t *testing.T, dbType test.DBType) (*base.BaseDendrite, f cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(filepath.Join("file://", tempDir, "userapi.db")) cfg.RelayAPI.Database.ConnectionString = config.DataSource(filepath.Join("file://", tempDir, "relayapi.db")) - base := base.NewBaseDendrite(&cfg, base.DisableMetrics) - return base, func() { - base.ShutdownDendrite() - base.WaitForShutdown() + return &cfg, ctx, func() { + ctx.ShutdownDendrite() + ctx.WaitForShutdown() t.Cleanup(func() {}) // removes t.TempDir, where all database files are created } default: t.Fatalf("unknown db type: %v", dbType) } - return nil, nil -} - -func Base(cfg *config.Dendrite) (*base.BaseDendrite, nats.JetStreamContext, *nats.Conn) { - if cfg == nil { - cfg = &config.Dendrite{} - cfg.Defaults(config.DefaultOpts{ - Generate: true, - SingleDatabase: false, - }) - } - cfg.Global.JetStream.InMemory = true - cfg.SyncAPI.Fulltext.InMemory = true - cfg.FederationAPI.KeyPerspectives = nil - base := base.NewBaseDendrite(cfg, base.DisableMetrics) - js, jc := base.NATS.Prepare(base.ProcessContext, &cfg.Global.JetStream) - return base, js, jc + return &config.Dendrite{}, nil, func() {} } diff --git a/test/testrig/jetstream.go b/test/testrig/jetstream.go index b880eea439..5f15cfb3ea 100644 --- a/test/testrig/jetstream.go +++ b/test/testrig/jetstream.go @@ -4,10 +4,10 @@ import ( "encoding/json" "testing" + "github.com/matrix-org/dendrite/setup/config" "github.com/nats-io/nats.go" "github.com/matrix-org/dendrite/roomserver/api" - "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/jetstream" ) @@ -20,9 +20,9 @@ func MustPublishMsgs(t *testing.T, jsctx nats.JetStreamContext, msgs ...*nats.Ms } } -func NewOutputEventMsg(t *testing.T, base *base.BaseDendrite, roomID string, update api.OutputEvent) *nats.Msg { +func NewOutputEventMsg(t *testing.T, cfg *config.Dendrite, roomID string, update api.OutputEvent) *nats.Msg { t.Helper() - msg := nats.NewMsg(base.Cfg.Global.JetStream.Prefixed(jetstream.OutputRoomEvent)) + msg := nats.NewMsg(cfg.Global.JetStream.Prefixed(jetstream.OutputRoomEvent)) msg.Header.Set(jetstream.RoomEventType, string(update.Type)) msg.Header.Set(jetstream.RoomID, roomID) var err error diff --git a/userapi/consumers/roomserver_test.go b/userapi/consumers/roomserver_test.go index a54706dbbc..4827ad47c2 100644 --- a/userapi/consumers/roomserver_test.go +++ b/userapi/consumers/roomserver_test.go @@ -21,7 +21,7 @@ import ( func mustCreateDatabase(t *testing.T, dbType test.DBType) (storage.UserDatabase, func()) { t.Helper() connStr, close := test.PrepareDBConnectionString(t, dbType) - cm := sqlutil.NewConnectionManager() + cm := sqlutil.NewConnectionManager(nil, config.DatabaseOptions{}) db, err := storage.NewUserDatabase(context.Background(), cm, &config.DatabaseOptions{ ConnectionString: config.DataSource(connStr), }, "", 4, 0, 0, "") diff --git a/userapi/internal/device_list_update_test.go b/userapi/internal/device_list_update_test.go index 6ccc4b5fbc..c0965a2c28 100644 --- a/userapi/internal/device_list_update_test.go +++ b/userapi/internal/device_list_update_test.go @@ -364,7 +364,7 @@ func mustCreateKeyserverDB(t *testing.T, dbType test.DBType) (storage.KeyDatabas t.Helper() connStr, clearDB := test.PrepareDBConnectionString(t, dbType) - cm := sqlutil.NewConnectionManager() + cm := sqlutil.NewConnectionManager(nil, config.DatabaseOptions{}) db, err := storage.NewKeyDatabase(cm, &config.DatabaseOptions{ConnectionString: config.DataSource(connStr)}) if err != nil { t.Fatal(err) diff --git a/userapi/internal/key_api_test.go b/userapi/internal/key_api_test.go index ec315c738b..de2a6d2c84 100644 --- a/userapi/internal/key_api_test.go +++ b/userapi/internal/key_api_test.go @@ -16,7 +16,7 @@ import ( func mustCreateDatabase(t *testing.T, dbType test.DBType) (storage.KeyDatabase, func()) { t.Helper() connStr, close := test.PrepareDBConnectionString(t, dbType) - cm := sqlutil.NewConnectionManager() + cm := sqlutil.NewConnectionManager(nil, config.DatabaseOptions{}) db, err := storage.NewKeyDatabase(cm, &config.DatabaseOptions{ ConnectionString: config.DataSource(connStr), }) diff --git a/userapi/storage/storage_test.go b/userapi/storage/storage_test.go index c6369ef932..cf7c5144eb 100644 --- a/userapi/storage/storage_test.go +++ b/userapi/storage/storage_test.go @@ -35,7 +35,7 @@ var ( func mustCreateUserDatabase(t *testing.T, dbType test.DBType) (storage.UserDatabase, func()) { connStr, close := test.PrepareDBConnectionString(t, dbType) - cm := sqlutil.NewConnectionManager() + cm := sqlutil.NewConnectionManager(nil, config.DatabaseOptions{}) db, err := storage.NewUserDatabase(context.Background(), cm, &config.DatabaseOptions{ ConnectionString: config.DataSource(connStr), }, "localhost", bcrypt.MinCost, openIDLifetimeMS, loginTokenLifetime, "_server") @@ -576,8 +576,9 @@ func Test_Notification(t *testing.T) { } func mustCreateKeyDatabase(t *testing.T, dbType test.DBType) (storage.KeyDatabase, func()) { - base, close := testrig.CreateBaseDendrite(t, dbType) - db, err := storage.NewKeyDatabase(base.ConnectionManager, &base.Cfg.KeyServer.Database) + cfg, processCtx, close := testrig.CreateConfig(t, dbType) + cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) + db, err := storage.NewKeyDatabase(cm, &cfg.KeyServer.Database) if err != nil { t.Fatalf("failed to create new database: %v", err) } diff --git a/userapi/userapi.go b/userapi/userapi.go index 3ada8020c6..6dcbc121f5 100644 --- a/userapi/userapi.go +++ b/userapi/userapi.go @@ -19,10 +19,12 @@ import ( fedsenderapi "github.com/matrix-org/dendrite/federationapi/api" "github.com/matrix-org/dendrite/internal/pushgateway" + "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/dendrite/setup/process" "github.com/sirupsen/logrus" rsapi "github.com/matrix-org/dendrite/roomserver/api" - "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/jetstream" "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/dendrite/userapi/consumers" @@ -35,31 +37,33 @@ import ( // NewInternalAPI returns a concrete implementation of the internal API. Callers // can call functions directly on the returned API or via an HTTP interface using AddInternalRoutes. func NewInternalAPI( - base *base.BaseDendrite, + processContext *process.ProcessContext, + dendriteCfg *config.Dendrite, + cm sqlutil.Connections, + natsInstance *jetstream.NATSInstance, rsAPI rsapi.UserRoomserverAPI, fedClient fedsenderapi.KeyserverFederationAPI, ) *internal.UserInternalAPI { - cfg := &base.Cfg.UserAPI - js, _ := base.NATS.Prepare(base.ProcessContext, &cfg.Matrix.JetStream) - appServices := base.Cfg.Derived.ApplicationServices + js, _ := natsInstance.Prepare(processContext, &dendriteCfg.Global.JetStream) + appServices := dendriteCfg.Derived.ApplicationServices - pgClient := pushgateway.NewHTTPClient(cfg.PushGatewayDisableTLSValidation) + pgClient := pushgateway.NewHTTPClient(dendriteCfg.UserAPI.PushGatewayDisableTLSValidation) db, err := storage.NewUserDatabase( - base.ProcessContext.Context(), - base.ConnectionManager, - &cfg.AccountDatabase, - cfg.Matrix.ServerName, - cfg.BCryptCost, - cfg.OpenIDTokenLifetimeMS, + processContext.Context(), + cm, + &dendriteCfg.UserAPI.AccountDatabase, + dendriteCfg.Global.ServerName, + dendriteCfg.UserAPI.BCryptCost, + dendriteCfg.UserAPI.OpenIDTokenLifetimeMS, api.DefaultLoginTokenLifetime, - cfg.Matrix.ServerNotices.LocalPart, + dendriteCfg.UserAPI.Matrix.ServerNotices.LocalPart, ) if err != nil { logrus.WithError(err).Panicf("failed to connect to accounts db") } - keyDB, err := storage.NewKeyDatabase(base.ConnectionManager, &base.Cfg.KeyServer.Database) + keyDB, err := storage.NewKeyDatabase(cm, &dendriteCfg.KeyServer.Database) if err != nil { logrus.WithError(err).Panicf("failed to connect to key db") } @@ -70,11 +74,11 @@ func NewInternalAPI( // it's handled by clientapi, and hence uses its topic. When user // API handles it for all account data, we can remove it from // here. - cfg.Matrix.JetStream.Prefixed(jetstream.OutputClientData), - cfg.Matrix.JetStream.Prefixed(jetstream.OutputNotificationData), + dendriteCfg.Global.JetStream.Prefixed(jetstream.OutputClientData), + dendriteCfg.Global.JetStream.Prefixed(jetstream.OutputNotificationData), ) keyChangeProducer := &producers.KeyChange{ - Topic: cfg.Matrix.JetStream.Prefixed(jetstream.OutputKeyChangeEvent), + Topic: dendriteCfg.Global.JetStream.Prefixed(jetstream.OutputKeyChangeEvent), JetStream: js, DB: keyDB, } @@ -84,15 +88,15 @@ func NewInternalAPI( KeyDatabase: keyDB, SyncProducer: syncProducer, KeyChangeProducer: keyChangeProducer, - Config: cfg, + Config: &dendriteCfg.UserAPI, AppServices: appServices, RSAPI: rsAPI, - DisableTLSValidation: cfg.PushGatewayDisableTLSValidation, + DisableTLSValidation: dendriteCfg.UserAPI.PushGatewayDisableTLSValidation, PgClient: pgClient, FedClient: fedClient, } - updater := internal.NewDeviceListUpdater(base.ProcessContext, keyDB, userAPI, keyChangeProducer, fedClient, 8, rsAPI, cfg.Matrix.ServerName) // 8 workers TODO: configurable + updater := internal.NewDeviceListUpdater(processContext, keyDB, userAPI, keyChangeProducer, fedClient, 8, rsAPI, dendriteCfg.Global.ServerName) // 8 workers TODO: configurable userAPI.Updater = updater // Remove users which we don't share a room with anymore if err := updater.CleanUp(); err != nil { @@ -106,28 +110,28 @@ func NewInternalAPI( }() dlConsumer := consumers.NewDeviceListUpdateConsumer( - base.ProcessContext, cfg, js, updater, + processContext, &dendriteCfg.UserAPI, js, updater, ) if err := dlConsumer.Start(); err != nil { logrus.WithError(err).Panic("failed to start device list consumer") } sigConsumer := consumers.NewSigningKeyUpdateConsumer( - base.ProcessContext, cfg, js, userAPI, + processContext, &dendriteCfg.UserAPI, js, userAPI, ) if err := sigConsumer.Start(); err != nil { logrus.WithError(err).Panic("failed to start signing key consumer") } receiptConsumer := consumers.NewOutputReceiptEventConsumer( - base.ProcessContext, cfg, js, db, syncProducer, pgClient, + processContext, &dendriteCfg.UserAPI, js, db, syncProducer, pgClient, ) if err := receiptConsumer.Start(); err != nil { logrus.WithError(err).Panic("failed to start user API receipt consumer") } eventConsumer := consumers.NewOutputRoomEventConsumer( - base.ProcessContext, cfg, js, db, pgClient, rsAPI, syncProducer, + processContext, &dendriteCfg.UserAPI, js, db, pgClient, rsAPI, syncProducer, ) if err := eventConsumer.Start(); err != nil { logrus.WithError(err).Panic("failed to start user API streamed event consumer") @@ -136,15 +140,15 @@ func NewInternalAPI( var cleanOldNotifs func() cleanOldNotifs = func() { logrus.Infof("Cleaning old notifications") - if err := db.DeleteOldNotifications(base.Context()); err != nil { + if err := db.DeleteOldNotifications(processContext.Context()); err != nil { logrus.WithError(err).Error("Failed to clean old notifications") } time.AfterFunc(time.Hour, cleanOldNotifs) } time.AfterFunc(time.Minute, cleanOldNotifs) - if base.Cfg.Global.ReportStats.Enabled { - go util.StartPhoneHomeCollector(time.Now(), base.Cfg, db) + if dendriteCfg.Global.ReportStats.Enabled { + go util.StartPhoneHomeCollector(time.Now(), dendriteCfg, db) } return userAPI diff --git a/userapi/userapi_test.go b/userapi/userapi_test.go index c2d4e5a270..03e6563545 100644 --- a/userapi/userapi_test.go +++ b/userapi/userapi_test.go @@ -68,34 +68,25 @@ func MustMakeInternalAPI(t *testing.T, opts apiTestOpts, dbType test.DBType, pub if opts.loginTokenLifetime == 0 { opts.loginTokenLifetime = api.DefaultLoginTokenLifetime * time.Millisecond } - base, baseclose := testrig.CreateBaseDendrite(t, dbType) - connStr, close := test.PrepareDBConnectionString(t, dbType) + cfg, ctx, close := testrig.CreateConfig(t, dbType) sName := serverName if opts.serverName != "" { sName = gomatrixserverlib.ServerName(opts.serverName) } - cm := sqlutil.NewConnectionManager() - ctx := context.Background() - accountDB, err := storage.NewUserDatabase(ctx, cm, &config.DatabaseOptions{ - ConnectionString: config.DataSource(connStr), - }, sName, bcrypt.MinCost, config.DefaultOpenIDTokenLifetimeMS, opts.loginTokenLifetime, "") + cm := sqlutil.NewConnectionManager(ctx, cfg.Global.DatabaseOptions) + + accountDB, err := storage.NewUserDatabase(ctx.Context(), cm, &cfg.UserAPI.AccountDatabase, sName, bcrypt.MinCost, config.DefaultOpenIDTokenLifetimeMS, opts.loginTokenLifetime, "") if err != nil { t.Fatalf("failed to create account DB: %s", err) } - keyDB, err := storage.NewKeyDatabase(base.ConnectionManager, &config.DatabaseOptions{ - ConnectionString: config.DataSource(connStr), - }) + keyDB, err := storage.NewKeyDatabase(cm, &cfg.KeyServer.Database) if err != nil { t.Fatalf("failed to create key DB: %s", err) } - cfg := &config.UserAPI{ - Matrix: &config.Global{ - SigningIdentity: gomatrixserverlib.SigningIdentity{ - ServerName: sName, - }, - }, + cfg.Global.SigningIdentity = gomatrixserverlib.SigningIdentity{ + ServerName: sName, } if publisher == nil { @@ -107,12 +98,11 @@ func MustMakeInternalAPI(t *testing.T, opts apiTestOpts, dbType test.DBType, pub return &internal.UserInternalAPI{ DB: accountDB, KeyDatabase: keyDB, - Config: cfg, + Config: &cfg.UserAPI, SyncProducer: syncProducer, KeyChangeProducer: keyChangeProducer, }, accountDB, func() { close() - baseclose() } } diff --git a/userapi/util/notify_test.go b/userapi/util/notify_test.go index c899e3a7c0..69461ddd1e 100644 --- a/userapi/util/notify_test.go +++ b/userapi/util/notify_test.go @@ -77,7 +77,7 @@ func TestNotifyUserCountsAsync(t *testing.T) { // Create DB and Dendrite base connStr, close := test.PrepareDBConnectionString(t, dbType) defer close() - cm := sqlutil.NewConnectionManager() + cm := sqlutil.NewConnectionManager(nil, config.DatabaseOptions{}) db, err := storage.NewUserDatabase(ctx, cm, &config.DatabaseOptions{ ConnectionString: config.DataSource(connStr), }, "test", bcrypt.MinCost, 0, 0, "") diff --git a/userapi/util/phonehomestats_test.go b/userapi/util/phonehomestats_test.go index 4e931a1b7f..191a35c044 100644 --- a/userapi/util/phonehomestats_test.go +++ b/userapi/util/phonehomestats_test.go @@ -7,10 +7,10 @@ import ( "testing" "time" + "github.com/matrix-org/dendrite/internal/sqlutil" "golang.org/x/crypto/bcrypt" "github.com/matrix-org/dendrite/internal" - "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/test" "github.com/matrix-org/dendrite/test/testrig" "github.com/matrix-org/dendrite/userapi/storage" @@ -18,12 +18,10 @@ import ( func TestCollect(t *testing.T) { test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { - b, _, _ := testrig.Base(nil) - connStr, closeDB := test.PrepareDBConnectionString(t, dbType) + cfg, processCtx, closeDB := testrig.CreateConfig(t, dbType) defer closeDB() - db, err := storage.NewUserDatabase(b.Context(), b.ConnectionManager, &config.DatabaseOptions{ - ConnectionString: config.DataSource(connStr), - }, "localhost", bcrypt.MinCost, 1000, 1000, "") + cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) + db, err := storage.NewUserDatabase(processCtx.Context(), cm, &cfg.UserAPI.AccountDatabase, "localhost", bcrypt.MinCost, 1000, 1000, "") if err != nil { t.Error(err) } @@ -62,12 +60,12 @@ func TestCollect(t *testing.T) { })) defer srv.Close() - b.Cfg.Global.ReportStats.Endpoint = srv.URL + cfg.Global.ReportStats.Endpoint = srv.URL stats := phoneHomeStats{ prevData: timestampToRUUsage{}, serverName: "localhost", startTime: time.Now(), - cfg: b.Cfg, + cfg: cfg, db: db, isMonolith: false, client: &http.Client{Timeout: time.Second}, From 6948d1652742ae83cf7729f1cfaa0d1ea796b703 Mon Sep 17 00:00:00 2001 From: Till Faelligen <2353100+S7evinK@users.noreply.github.com> Date: Wed, 22 Mar 2023 14:12:06 +0100 Subject: [PATCH 07/42] Update Go, use go tool covdata for coverage files? --- .github/workflows/schedules.yaml | 19 +++++++++---------- build/scripts/Complement.Dockerfile | 4 ++-- build/scripts/ComplementPostgres.Dockerfile | 10 +++++----- build/scripts/complement-cmd.sh | 5 +++-- internal/fulltext/bleve.go | 12 +++++++----- internal/fulltext/bleve_test.go | 2 +- syncapi/syncapi.go | 3 +-- test/testrig/base.go | 3 ++- 8 files changed, 30 insertions(+), 28 deletions(-) diff --git a/.github/workflows/schedules.yaml b/.github/workflows/schedules.yaml index dff9b34c9b..b6a15bab07 100644 --- a/.github/workflows/schedules.yaml +++ b/.github/workflows/schedules.yaml @@ -12,6 +12,7 @@ concurrency: jobs: # run Sytest in different variations sytest: + if: ${{ false }} timeout-minutes: 60 name: "Sytest (${{ matrix.label }})" runs-on: ubuntu-latest @@ -75,7 +76,7 @@ jobs: name: "Sytest Coverage" runs-on: ubuntu-latest needs: sytest # only run once Sytest is done - if: ${{ always() }} + if: ${{ false }} steps: - uses: actions/checkout@v3 - name: Install Go @@ -167,7 +168,7 @@ jobs: cat < /tmp/posttest.sh #!/bin/bash mkdir -p /tmp/Complement/logs/\$2/\$1/ - docker cp \$1:/dendrite/complementcover.log /tmp/Complement/logs/\$2/\$1/ + docker cp \$1:/tmp/covdatafiles/. /tmp/Complement/logs/\$2/\$1/ EOF chmod +x /tmp/posttest.sh @@ -190,7 +191,7 @@ jobs: with: name: Complement Logs - (Dendrite, ${{ join(matrix.*, ', ') }}) path: | - /tmp/Complement/**/complementcover.log + /tmp/Complement/logs/** complement-coverage: timeout-minutes: 5 @@ -199,7 +200,6 @@ jobs: needs: complement # only run once Complement is done if: ${{ always() }} steps: - - uses: actions/checkout@v3 - name: Install Go uses: actions/setup-go@v3 with: @@ -207,20 +207,19 @@ jobs: cache: true - name: Download all artifacts uses: actions/download-artifact@v3 - - name: Install gocovmerge - run: go install github.com/wadey/gocovmerge@latest - - name: Run gocovmerge + - name: Collect coverage run: | - find -name 'complementcover.log' -printf '"%p"\n' | xargs gocovmerge | grep -Ev 'relayapi|setup/mscs|api_trace' > complement.cov - go tool cover -func=complement.cov + go tool covdata textfmt -i=$(find . -name 'covmeta*' -type f -exec dirname {} \; | uniq | awk -F '/' '$3!="" {print $0}' | paste -s -d ',' -) -o complement.cov + grep -Ev 'relayapi|setup/mscs|api_trace' complement.cov > final.cov - name: Upload coverage to Codecov uses: codecov/codecov-action@v3 with: - files: ./complement.cov + files: ./final.cov flags: complement fail_ci_if_error: true element_web: + if: ${{ false }} timeout-minutes: 120 runs-on: ubuntu-latest steps: diff --git a/build/scripts/Complement.Dockerfile b/build/scripts/Complement.Dockerfile index 70bbe8f95a..453d897658 100644 --- a/build/scripts/Complement.Dockerfile +++ b/build/scripts/Complement.Dockerfile @@ -1,6 +1,6 @@ #syntax=docker/dockerfile:1.2 -FROM golang:1.18-stretch as build +FROM golang:1.20-bullseye as build RUN apt-get update && apt-get install -y sqlite3 WORKDIR /build @@ -17,7 +17,7 @@ RUN --mount=target=. \ CGO_ENABLED=${CGO} go build -o /dendrite ./cmd/generate-config && \ CGO_ENABLED=${CGO} go build -o /dendrite ./cmd/generate-keys && \ CGO_ENABLED=${CGO} go build -o /dendrite/dendrite ./cmd/dendrite && \ - CGO_ENABLED=${CGO} go test -c -cover -covermode=atomic -o /dendrite/dendrite-cover -coverpkg "github.com/matrix-org/..." ./cmd/dendrite && \ + CGO_ENABLED=${CGO} go build -cover -covermode=atomic -o /dendrite/dendrite-cover -coverpkg "github.com/matrix-org/..." ./cmd/dendrite && \ cp build/scripts/complement-cmd.sh /complement-cmd.sh WORKDIR /dendrite diff --git a/build/scripts/ComplementPostgres.Dockerfile b/build/scripts/ComplementPostgres.Dockerfile index d4b6d3f75b..77071b4506 100644 --- a/build/scripts/ComplementPostgres.Dockerfile +++ b/build/scripts/ComplementPostgres.Dockerfile @@ -1,19 +1,19 @@ #syntax=docker/dockerfile:1.2 -FROM golang:1.18-stretch as build +FROM golang:1.20-bullseye as build RUN apt-get update && apt-get install -y postgresql WORKDIR /build # No password when connecting over localhost -RUN sed -i "s%127.0.0.1/32 md5%127.0.0.1/32 trust%g" /etc/postgresql/9.6/main/pg_hba.conf && \ +RUN sed -i "s%127.0.0.1/32 md5%127.0.0.1/32 trust%g" /etc/postgresql/13/main/pg_hba.conf && \ # Bump up max conns for moar concurrency - sed -i 's/max_connections = 100/max_connections = 2000/g' /etc/postgresql/9.6/main/postgresql.conf + sed -i 's/max_connections = 100/max_connections = 2000/g' /etc/postgresql/13/main/postgresql.conf # This entry script starts postgres, waits for it to be up then starts dendrite RUN echo '\ #!/bin/bash -eu \n\ pg_lsclusters \n\ - pg_ctlcluster 9.6 main start \n\ + pg_ctlcluster 13 main start \n\ \n\ until pg_isready \n\ do \n\ @@ -35,7 +35,7 @@ RUN --mount=target=. \ CGO_ENABLED=${CGO} go build -o /dendrite ./cmd/generate-config && \ CGO_ENABLED=${CGO} go build -o /dendrite ./cmd/generate-keys && \ CGO_ENABLED=${CGO} go build -o /dendrite/dendrite ./cmd/dendrite && \ - CGO_ENABLED=${CGO} go test -c -cover -covermode=atomic -o /dendrite/dendrite-cover -coverpkg "github.com/matrix-org/..." ./cmd/dendrite && \ + CGO_ENABLED=${CGO} go build -cover -covermode=atomic -o /dendrite/dendrite-cover -coverpkg "github.com/matrix-org/..." ./cmd/dendrite && \ cp build/scripts/complement-cmd.sh /complement-cmd.sh WORKDIR /dendrite diff --git a/build/scripts/complement-cmd.sh b/build/scripts/complement-cmd.sh index 52b063d01a..3f6ed070cd 100755 --- a/build/scripts/complement-cmd.sh +++ b/build/scripts/complement-cmd.sh @@ -2,14 +2,15 @@ # This script is intended to be used inside a docker container for Complement +export GOCOVERDIR=/tmp/covdatafiles +mkdir -p "${GOCOVERDIR}" if [[ "${COVER}" -eq 1 ]]; then echo "Running with coverage" exec /dendrite/dendrite-cover \ --really-enable-open-registration \ --tls-cert server.crt \ --tls-key server.key \ - --config dendrite.yaml \ - --test.coverprofile=complementcover.log + --config dendrite.yaml else echo "Not running with coverage" exec /dendrite/dendrite \ diff --git a/internal/fulltext/bleve.go b/internal/fulltext/bleve.go index 08cef8a97f..dea7c504c6 100644 --- a/internal/fulltext/bleve.go +++ b/internal/fulltext/bleve.go @@ -18,10 +18,11 @@ package fulltext import ( - "context" "strings" "github.com/blevesearch/bleve/v2" + "github.com/matrix-org/dendrite/setup/process" + // side effect imports to allow all possible languages _ "github.com/blevesearch/bleve/v2/analysis/lang/ar" _ "github.com/blevesearch/bleve/v2/analysis/lang/cjk" @@ -84,17 +85,18 @@ func (i *IndexElement) SetContentType(v string) { } // New opens a new/existing fulltext index -func New(ctx context.Context, cfg config.Fulltext) (fts *Search, err error) { +func New(processCtx *process.ProcessContext, cfg config.Fulltext) (fts *Search, err error) { fts = &Search{} fts.FulltextIndex, err = openIndex(cfg) if err != nil { return nil, err } go func() { - // Wait for the context (should be from process.ProcessContext) to be - // done, indicating that Dendrite is shutting down. - <-ctx.Done() + processCtx.ComponentStarted() + // Wait for the processContext to be done, indicating that Dendrite is shutting down. + <-processCtx.WaitForShutdown() _ = fts.Close() + processCtx.ComponentFinished() }() return fts, nil } diff --git a/internal/fulltext/bleve_test.go b/internal/fulltext/bleve_test.go index 6ce6914548..bd8289d58b 100644 --- a/internal/fulltext/bleve_test.go +++ b/internal/fulltext/bleve_test.go @@ -38,7 +38,7 @@ func mustOpenIndex(t *testing.T, tempDir string) (*fulltext.Search, *process.Pro cfg.InMemory = false } ctx := process.NewProcessContext() - fts, err := fulltext.New(ctx.Context(), cfg) + fts, err := fulltext.New(ctx, cfg) if err != nil { t.Fatal("failed to open fulltext index:", err) } diff --git a/syncapi/syncapi.go b/syncapi/syncapi.go index 9a27b954a4..ecbe05dd84 100644 --- a/syncapi/syncapi.go +++ b/syncapi/syncapi.go @@ -69,11 +69,10 @@ func AddPublicRoutes( var fts *fulltext.Search if dendriteCfg.SyncAPI.Fulltext.Enabled { - fts, err = fulltext.New(processContext.Context(), dendriteCfg.SyncAPI.Fulltext) + fts, err = fulltext.New(processContext, dendriteCfg.SyncAPI.Fulltext) if err != nil { logrus.WithError(err).Panicf("failed to create full text") } - processContext.ComponentStarted() } federationPresenceProducer := &producers.FederationAPIPresenceProducer{ diff --git a/test/testrig/base.go b/test/testrig/base.go index fd85578ee3..9537045959 100644 --- a/test/testrig/base.go +++ b/test/testrig/base.go @@ -51,6 +51,7 @@ func CreateConfig(t *testing.T, dbType test.DBType) (*config.Dendrite, *process. // use a distinct prefix else concurrent postgres/sqlite runs will clash since NATS will use // the file system event with InMemory=true :( cfg.Global.JetStream.TopicPrefix = fmt.Sprintf("Test_%d_", dbType) + cfg.SyncAPI.Fulltext.InMemory = true connStr, closeDb := test.PrepareDBConnectionString(t, dbType) cfg.Global.DatabaseOptions = config.DatabaseOptions{ @@ -70,7 +71,7 @@ func CreateConfig(t *testing.T, dbType test.DBType) (*config.Dendrite, *process. SingleDatabase: false, }) cfg.Global.ServerName = "test" - + cfg.SyncAPI.Fulltext.InMemory = true // use a distinct prefix else concurrent postgres/sqlite runs will clash since NATS will use // the file system event with InMemory=true :( cfg.Global.JetStream.TopicPrefix = fmt.Sprintf("Test_%d_", dbType) From b741d38e104c45b63630ac1068ed3d425f690691 Mon Sep 17 00:00:00 2001 From: Till Faelligen <2353100+S7evinK@users.noreply.github.com> Date: Wed, 22 Mar 2023 14:51:18 +0100 Subject: [PATCH 08/42] Update Workflow --- .github/workflows/schedules.yaml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/schedules.yaml b/.github/workflows/schedules.yaml index b6a15bab07..afbefbdd2b 100644 --- a/.github/workflows/schedules.yaml +++ b/.github/workflows/schedules.yaml @@ -12,7 +12,6 @@ concurrency: jobs: # run Sytest in different variations sytest: - if: ${{ false }} timeout-minutes: 60 name: "Sytest (${{ matrix.label }})" runs-on: ubuntu-latest @@ -70,6 +69,7 @@ jobs: path: | /logs/results.tap /logs/**/*.log* + /logs/covdatafiles/** sytest-coverage: timeout-minutes: 5 @@ -86,16 +86,15 @@ jobs: cache: true - name: Download all artifacts uses: actions/download-artifact@v3 - - name: Install gocovmerge - run: go install github.com/wadey/gocovmerge@latest - - name: Run gocovmerge + - name: Collect coverage run: | - find -name 'integrationcover.log' -printf '"%p"\n' | xargs gocovmerge | grep -Ev 'relayapi|setup/mscs|api_trace' > sytest.cov - go tool cover -func=sytest.cov + go tool covdata textfmt -i=$(find Sytest* -name 'covmeta*' -type f -exec dirname {} \; | uniq | awk -F '/' '$3!="" {print $0}' | paste -s -d ',' -) -o sytest.cov + grep -Ev 'relayapi|setup/mscs|api_trace' complement.cov > final.cov + go tool covdata func -i=$(find Sytest* -name 'covmeta*' -type f -exec dirname {} \; | uniq | awk -F '/' '$3!="" {print $0}' | paste -s -d ',' -) - name: Upload coverage to Codecov uses: codecov/codecov-action@v3 with: - files: ./sytest.cov + files: ./final.cov flags: sytest fail_ci_if_error: true @@ -200,6 +199,7 @@ jobs: needs: complement # only run once Complement is done if: ${{ always() }} steps: + - uses: actions/checkout@v3 - name: Install Go uses: actions/setup-go@v3 with: @@ -209,8 +209,9 @@ jobs: uses: actions/download-artifact@v3 - name: Collect coverage run: | - go tool covdata textfmt -i=$(find . -name 'covmeta*' -type f -exec dirname {} \; | uniq | awk -F '/' '$3!="" {print $0}' | paste -s -d ',' -) -o complement.cov + go tool covdata textfmt -i=$(find Complement* -name 'covmeta*' -type f -exec dirname {} \; | uniq | awk -F '/' '$3!="" {print $0}' | paste -s -d ',' -) -o complement.cov grep -Ev 'relayapi|setup/mscs|api_trace' complement.cov > final.cov + go tool covdata func -i=$(find Complement* -name 'covmeta*' -type f -exec dirname {} \; | uniq | awk -F '/' '$3!="" {print $0}' | paste -s -d ',' -) - name: Upload coverage to Codecov uses: codecov/codecov-action@v3 with: @@ -219,7 +220,6 @@ jobs: fail_ci_if_error: true element_web: - if: ${{ false }} timeout-minutes: 120 runs-on: ubuntu-latest steps: From a4400bdd765659162f2b1353694828450513ab5c Mon Sep 17 00:00:00 2001 From: Till Faelligen <2353100+S7evinK@users.noreply.github.com> Date: Wed, 22 Mar 2023 14:58:36 +0100 Subject: [PATCH 09/42] Sytest coverage file --- .github/workflows/schedules.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/schedules.yaml b/.github/workflows/schedules.yaml index afbefbdd2b..1606b0a75c 100644 --- a/.github/workflows/schedules.yaml +++ b/.github/workflows/schedules.yaml @@ -89,7 +89,7 @@ jobs: - name: Collect coverage run: | go tool covdata textfmt -i=$(find Sytest* -name 'covmeta*' -type f -exec dirname {} \; | uniq | awk -F '/' '$3!="" {print $0}' | paste -s -d ',' -) -o sytest.cov - grep -Ev 'relayapi|setup/mscs|api_trace' complement.cov > final.cov + grep -Ev 'relayapi|setup/mscs|api_trace' sytest.cov > final.cov go tool covdata func -i=$(find Sytest* -name 'covmeta*' -type f -exec dirname {} \; | uniq | awk -F '/' '$3!="" {print $0}' | paste -s -d ',' -) - name: Upload coverage to Codecov uses: codecov/codecov-action@v3 From 14085d30ac161b01a87d99d4c73813344a7e349d Mon Sep 17 00:00:00 2001 From: Till Faelligen <2353100+S7evinK@users.noreply.github.com> Date: Wed, 22 Mar 2023 16:01:12 +0100 Subject: [PATCH 10/42] Update workflow to not use commas when joining names --- .github/workflows/schedules.yaml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/schedules.yaml b/.github/workflows/schedules.yaml index 1606b0a75c..a97434695e 100644 --- a/.github/workflows/schedules.yaml +++ b/.github/workflows/schedules.yaml @@ -65,7 +65,7 @@ jobs: uses: actions/upload-artifact@v2 if: ${{ always() }} with: - name: Sytest Logs - ${{ job.status }} - (Dendrite, ${{ join(matrix.*, ', ') }}) + name: Sytest Logs - ${{ job.status }} - (Dendrite ${{ join(matrix.*, ' ') }}) path: | /logs/results.tap /logs/**/*.log* @@ -76,7 +76,7 @@ jobs: name: "Sytest Coverage" runs-on: ubuntu-latest needs: sytest # only run once Sytest is done - if: ${{ false }} + if: ${{ always() }} steps: - uses: actions/checkout@v3 - name: Install Go @@ -88,9 +88,9 @@ jobs: uses: actions/download-artifact@v3 - name: Collect coverage run: | - go tool covdata textfmt -i=$(find Sytest* -name 'covmeta*' -type f -exec dirname {} \; | uniq | awk -F '/' '$3!="" {print $0}' | paste -s -d ',' -) -o sytest.cov + go tool covdata textfmt -i="$(find Sytest* -name 'covmeta*' -type f -exec dirname {} \; | uniq | paste -s -d ',' -)" -o sytest.cov grep -Ev 'relayapi|setup/mscs|api_trace' sytest.cov > final.cov - go tool covdata func -i=$(find Sytest* -name 'covmeta*' -type f -exec dirname {} \; | uniq | awk -F '/' '$3!="" {print $0}' | paste -s -d ',' -) + go tool covdata func -i="$(find Sytest* -name 'covmeta*' -type f -exec dirname {} \; | uniq | paste -s -d ',' -)" - name: Upload coverage to Codecov uses: codecov/codecov-action@v3 with: @@ -188,7 +188,7 @@ jobs: uses: actions/upload-artifact@v2 if: ${{ always() }} with: - name: Complement Logs - (Dendrite, ${{ join(matrix.*, ', ') }}) + name: Complement Logs - (Dendrite ${{ join(matrix.*, ' ') }}) path: | /tmp/Complement/logs/** @@ -209,9 +209,9 @@ jobs: uses: actions/download-artifact@v3 - name: Collect coverage run: | - go tool covdata textfmt -i=$(find Complement* -name 'covmeta*' -type f -exec dirname {} \; | uniq | awk -F '/' '$3!="" {print $0}' | paste -s -d ',' -) -o complement.cov + go tool covdata textfmt -i="$(find Complement* -name 'covmeta*' -type f -exec dirname {} \; | uniq | paste -s -d ',' -)" -o complement.cov grep -Ev 'relayapi|setup/mscs|api_trace' complement.cov > final.cov - go tool covdata func -i=$(find Complement* -name 'covmeta*' -type f -exec dirname {} \; | uniq | awk -F '/' '$3!="" {print $0}' | paste -s -d ',' -) + go tool covdata func -i="$(find Complement* -name 'covmeta*' -type f -exec dirname {} \; | uniq | paste -s -d ',' -)" - name: Upload coverage to Codecov uses: codecov/codecov-action@v3 with: From cb18ba023084a364a332d09e32ae22d375ed4520 Mon Sep 17 00:00:00 2001 From: Till Faelligen <2353100+S7evinK@users.noreply.github.com> Date: Wed, 22 Mar 2023 17:36:33 +0100 Subject: [PATCH 11/42] Upload covdatafiles for each server --- .github/workflows/schedules.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/schedules.yaml b/.github/workflows/schedules.yaml index a97434695e..2545949122 100644 --- a/.github/workflows/schedules.yaml +++ b/.github/workflows/schedules.yaml @@ -69,7 +69,7 @@ jobs: path: | /logs/results.tap /logs/**/*.log* - /logs/covdatafiles/** + /logs/**/covdatafiles/** sytest-coverage: timeout-minutes: 5 From 234ed603e6b8c7f1f766a002a097007189e1184e Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Thu, 23 Mar 2023 13:52:53 +0100 Subject: [PATCH 12/42] Move every `db.Prepare` to `sqlutil.Statementlist`, remove trace driver (#3026) Doesn't buy us much, but makes everything a bit more consistent. Also removes the SQL trace driver, as it is unused and the output is hard to read anyway. --- docs/development/sytest.md | 1 - .../storage/postgres/blacklist_table.go | 19 +-- .../storage/postgres/joined_hosts_table.go | 31 ++--- .../postgres/notary_server_keys_json_table.go | 8 +- .../notary_server_keys_metadata_table.go | 24 ++-- .../storage/postgres/queue_json_table.go | 15 +-- .../storage/postgres/queue_pdus_table.go | 23 ++-- .../storage/postgres/server_key_table.go | 11 +- .../storage/sqlite3/blacklist_table.go | 19 +-- .../storage/sqlite3/joined_hosts_table.go | 24 ++-- .../sqlite3/notary_server_keys_json_table.go | 8 +- .../notary_server_keys_metadata_table.go | 19 +-- .../storage/sqlite3/queue_json_table.go | 8 +- .../storage/sqlite3/queue_pdus_table.go | 26 ++--- .../storage/sqlite3/server_key_table.go | 11 +- go.mod | 1 - go.sum | 2 - internal/sqlutil/sqlite_cgo.go | 5 - internal/sqlutil/sqlite_native.go | 5 - internal/sqlutil/sqlutil.go | 7 +- internal/sqlutil/trace.go | 109 ------------------ internal/sqlutil/trace_driver.go | 35 ------ internal/sqlutil/trace_driver_wasm.go | 34 ------ internal/sqlutil/writer_exclusive.go | 5 - .../storage/postgres/account_data_table.go | 15 +-- syncapi/storage/postgres/filter_table.go | 15 +-- syncapi/storage/postgres/ignores_table.go | 12 +- .../storage/postgres/send_to_device_table.go | 19 +-- syncapi/storage/sqlite3/account_data_table.go | 15 +-- syncapi/storage/sqlite3/filter_table.go | 15 +-- syncapi/storage/sqlite3/ignores_table.go | 11 +- .../storage/sqlite3/send_to_device_table.go | 19 +-- syncapi/storage/sqlite3/stream_id_table.go | 7 +- 33 files changed, 127 insertions(+), 451 deletions(-) delete mode 100644 internal/sqlutil/trace.go delete mode 100644 internal/sqlutil/trace_driver.go delete mode 100644 internal/sqlutil/trace_driver_wasm.go diff --git a/docs/development/sytest.md b/docs/development/sytest.md index 3cfb99e603..4fae2ea3d1 100644 --- a/docs/development/sytest.md +++ b/docs/development/sytest.md @@ -61,7 +61,6 @@ When debugging, the following Docker `run` options may also be useful: * `-e "DENDRITE_TRACE_HTTP=1"`: Adds HTTP tracing to server logs. * `-e "DENDRITE_TRACE_INTERNAL=1"`: Adds roomserver internal API tracing to server logs. -* `-e "DENDRITE_TRACE_SQL=1"`: Adds tracing to all SQL statements to server logs. The docker command also supports a single positional argument for the test file to run, so you can run a single `.pl` file rather than the whole test suite. For example: diff --git a/federationapi/storage/postgres/blacklist_table.go b/federationapi/storage/postgres/blacklist_table.go index eef37318bb..1d931daa3b 100644 --- a/federationapi/storage/postgres/blacklist_table.go +++ b/federationapi/storage/postgres/blacklist_table.go @@ -60,19 +60,12 @@ func NewPostgresBlacklistTable(db *sql.DB) (s *blacklistStatements, err error) { return } - if s.insertBlacklistStmt, err = db.Prepare(insertBlacklistSQL); err != nil { - return - } - if s.selectBlacklistStmt, err = db.Prepare(selectBlacklistSQL); err != nil { - return - } - if s.deleteBlacklistStmt, err = db.Prepare(deleteBlacklistSQL); err != nil { - return - } - if s.deleteAllBlacklistStmt, err = db.Prepare(deleteAllBlacklistSQL); err != nil { - return - } - return + return s, sqlutil.StatementList{ + {&s.insertBlacklistStmt, insertBlacklistSQL}, + {&s.selectBlacklistStmt, selectBlacklistSQL}, + {&s.deleteBlacklistStmt, deleteBlacklistSQL}, + {&s.deleteAllBlacklistStmt, deleteAllBlacklistSQL}, + }.Prepare(db) } func (s *blacklistStatements) InsertBlacklist( diff --git a/federationapi/storage/postgres/joined_hosts_table.go b/federationapi/storage/postgres/joined_hosts_table.go index 9a3977560b..8806db550c 100644 --- a/federationapi/storage/postgres/joined_hosts_table.go +++ b/federationapi/storage/postgres/joined_hosts_table.go @@ -90,28 +90,15 @@ func NewPostgresJoinedHostsTable(db *sql.DB) (s *joinedHostsStatements, err erro if err != nil { return } - if s.insertJoinedHostsStmt, err = s.db.Prepare(insertJoinedHostsSQL); err != nil { - return - } - if s.deleteJoinedHostsStmt, err = s.db.Prepare(deleteJoinedHostsSQL); err != nil { - return - } - if s.deleteJoinedHostsForRoomStmt, err = s.db.Prepare(deleteJoinedHostsForRoomSQL); err != nil { - return - } - if s.selectJoinedHostsStmt, err = s.db.Prepare(selectJoinedHostsSQL); err != nil { - return - } - if s.selectAllJoinedHostsStmt, err = s.db.Prepare(selectAllJoinedHostsSQL); err != nil { - return - } - if s.selectJoinedHostsForRoomsStmt, err = s.db.Prepare(selectJoinedHostsForRoomsSQL); err != nil { - return - } - if s.selectJoinedHostsForRoomsExcludingBlacklistedStmt, err = s.db.Prepare(selectJoinedHostsForRoomsExcludingBlacklistedSQL); err != nil { - return - } - return + return s, sqlutil.StatementList{ + {&s.insertJoinedHostsStmt, insertJoinedHostsSQL}, + {&s.deleteJoinedHostsStmt, deleteJoinedHostsSQL}, + {&s.deleteJoinedHostsForRoomStmt, deleteJoinedHostsForRoomSQL}, + {&s.selectJoinedHostsStmt, selectJoinedHostsSQL}, + {&s.selectAllJoinedHostsStmt, selectAllJoinedHostsSQL}, + {&s.selectJoinedHostsForRoomsStmt, selectJoinedHostsForRoomsSQL}, + {&s.selectJoinedHostsForRoomsExcludingBlacklistedStmt, selectJoinedHostsForRoomsExcludingBlacklistedSQL}, + }.Prepare(db) } func (s *joinedHostsStatements) InsertJoinedHosts( diff --git a/federationapi/storage/postgres/notary_server_keys_json_table.go b/federationapi/storage/postgres/notary_server_keys_json_table.go index 65221c088c..9fc93a612d 100644 --- a/federationapi/storage/postgres/notary_server_keys_json_table.go +++ b/federationapi/storage/postgres/notary_server_keys_json_table.go @@ -19,6 +19,7 @@ import ( "database/sql" "github.com/matrix-org/dendrite/federationapi/storage/tables" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" ) @@ -50,10 +51,9 @@ func NewPostgresNotaryServerKeysTable(db *sql.DB) (s *notaryServerKeysStatements return } - if s.insertServerKeysJSONStmt, err = db.Prepare(insertServerKeysJSONSQL); err != nil { - return - } - return + return s, sqlutil.StatementList{ + {&s.insertServerKeysJSONStmt, insertServerKeysJSONSQL}, + }.Prepare(db) } func (s *notaryServerKeysStatements) InsertJSONResponse( diff --git a/federationapi/storage/postgres/notary_server_keys_metadata_table.go b/federationapi/storage/postgres/notary_server_keys_metadata_table.go index 72faf4809b..6d38ccab5f 100644 --- a/federationapi/storage/postgres/notary_server_keys_metadata_table.go +++ b/federationapi/storage/postgres/notary_server_keys_metadata_table.go @@ -22,6 +22,7 @@ import ( "github.com/lib/pq" "github.com/matrix-org/dendrite/federationapi/storage/tables" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" ) @@ -91,22 +92,13 @@ func NewPostgresNotaryServerKeysMetadataTable(db *sql.DB) (s *notaryServerKeysMe return } - if s.upsertServerKeysStmt, err = db.Prepare(upsertServerKeysSQL); err != nil { - return - } - if s.selectNotaryKeyResponsesStmt, err = db.Prepare(selectNotaryKeyResponsesSQL); err != nil { - return - } - if s.selectNotaryKeyResponsesWithKeyIDsStmt, err = db.Prepare(selectNotaryKeyResponsesWithKeyIDsSQL); err != nil { - return - } - if s.selectNotaryKeyMetadataStmt, err = db.Prepare(selectNotaryKeyMetadataSQL); err != nil { - return - } - if s.deleteUnusedServerKeysJSONStmt, err = db.Prepare(deleteUnusedServerKeysJSONSQL); err != nil { - return - } - return + return s, sqlutil.StatementList{ + {&s.upsertServerKeysStmt, upsertServerKeysSQL}, + {&s.selectNotaryKeyResponsesStmt, selectNotaryKeyResponsesSQL}, + {&s.selectNotaryKeyResponsesWithKeyIDsStmt, selectNotaryKeyResponsesWithKeyIDsSQL}, + {&s.selectNotaryKeyMetadataStmt, selectNotaryKeyMetadataSQL}, + {&s.deleteUnusedServerKeysJSONStmt, deleteUnusedServerKeysJSONSQL}, + }.Prepare(db) } func (s *notaryServerKeysMetadataStatements) UpsertKey( diff --git a/federationapi/storage/postgres/queue_json_table.go b/federationapi/storage/postgres/queue_json_table.go index e330741823..563738dd58 100644 --- a/federationapi/storage/postgres/queue_json_table.go +++ b/federationapi/storage/postgres/queue_json_table.go @@ -65,16 +65,11 @@ func NewPostgresQueueJSONTable(db *sql.DB) (s *queueJSONStatements, err error) { if err != nil { return } - if s.insertJSONStmt, err = s.db.Prepare(insertJSONSQL); err != nil { - return - } - if s.deleteJSONStmt, err = s.db.Prepare(deleteJSONSQL); err != nil { - return - } - if s.selectJSONStmt, err = s.db.Prepare(selectJSONSQL); err != nil { - return - } - return + return s, sqlutil.StatementList{ + {&s.insertJSONStmt, insertJSONSQL}, + {&s.deleteJSONStmt, deleteJSONSQL}, + {&s.selectJSONStmt, selectJSONSQL}, + }.Prepare(db) } func (s *queueJSONStatements) InsertQueueJSON( diff --git a/federationapi/storage/postgres/queue_pdus_table.go b/federationapi/storage/postgres/queue_pdus_table.go index 3b0bef9af1..b97be4822c 100644 --- a/federationapi/storage/postgres/queue_pdus_table.go +++ b/federationapi/storage/postgres/queue_pdus_table.go @@ -78,22 +78,13 @@ func NewPostgresQueuePDUsTable(db *sql.DB) (s *queuePDUsStatements, err error) { if err != nil { return } - if s.insertQueuePDUStmt, err = s.db.Prepare(insertQueuePDUSQL); err != nil { - return - } - if s.deleteQueuePDUsStmt, err = s.db.Prepare(deleteQueuePDUSQL); err != nil { - return - } - if s.selectQueuePDUsStmt, err = s.db.Prepare(selectQueuePDUsSQL); err != nil { - return - } - if s.selectQueuePDUReferenceJSONCountStmt, err = s.db.Prepare(selectQueuePDUReferenceJSONCountSQL); err != nil { - return - } - if s.selectQueuePDUServerNamesStmt, err = s.db.Prepare(selectQueuePDUServerNamesSQL); err != nil { - return - } - return + return s, sqlutil.StatementList{ + {&s.insertQueuePDUStmt, insertQueuePDUSQL}, + {&s.deleteQueuePDUsStmt, deleteQueuePDUSQL}, + {&s.selectQueuePDUsStmt, selectQueuePDUsSQL}, + {&s.selectQueuePDUReferenceJSONCountStmt, selectQueuePDUReferenceJSONCountSQL}, + {&s.selectQueuePDUServerNamesStmt, selectQueuePDUServerNamesSQL}, + }.Prepare(db) } func (s *queuePDUsStatements) InsertQueuePDU( diff --git a/federationapi/storage/postgres/server_key_table.go b/federationapi/storage/postgres/server_key_table.go index 16e294e05c..f393351bb2 100644 --- a/federationapi/storage/postgres/server_key_table.go +++ b/federationapi/storage/postgres/server_key_table.go @@ -72,13 +72,10 @@ func NewPostgresServerSigningKeysTable(db *sql.DB) (s *serverSigningKeyStatement if err != nil { return } - if s.bulkSelectServerKeysStmt, err = db.Prepare(bulkSelectServerSigningKeysSQL); err != nil { - return - } - if s.upsertServerKeysStmt, err = db.Prepare(upsertServerSigningKeysSQL); err != nil { - return - } - return s, nil + return s, sqlutil.StatementList{ + {&s.bulkSelectServerKeysStmt, bulkSelectServerSigningKeysSQL}, + {&s.upsertServerKeysStmt, upsertServerSigningKeysSQL}, + }.Prepare(db) } func (s *serverSigningKeyStatements) BulkSelectServerKeys( diff --git a/federationapi/storage/sqlite3/blacklist_table.go b/federationapi/storage/sqlite3/blacklist_table.go index 2694e630d7..5122bff160 100644 --- a/federationapi/storage/sqlite3/blacklist_table.go +++ b/federationapi/storage/sqlite3/blacklist_table.go @@ -60,19 +60,12 @@ func NewSQLiteBlacklistTable(db *sql.DB) (s *blacklistStatements, err error) { return } - if s.insertBlacklistStmt, err = db.Prepare(insertBlacklistSQL); err != nil { - return - } - if s.selectBlacklistStmt, err = db.Prepare(selectBlacklistSQL); err != nil { - return - } - if s.deleteBlacklistStmt, err = db.Prepare(deleteBlacklistSQL); err != nil { - return - } - if s.deleteAllBlacklistStmt, err = db.Prepare(deleteAllBlacklistSQL); err != nil { - return - } - return + return s, sqlutil.StatementList{ + {&s.insertBlacklistStmt, insertBlacklistSQL}, + {&s.selectBlacklistStmt, selectBlacklistSQL}, + {&s.deleteBlacklistStmt, deleteBlacklistSQL}, + {&s.deleteAllBlacklistStmt, deleteAllBlacklistSQL}, + }.Prepare(db) } func (s *blacklistStatements) InsertBlacklist( diff --git a/federationapi/storage/sqlite3/joined_hosts_table.go b/federationapi/storage/sqlite3/joined_hosts_table.go index 83112c150f..2f0763829b 100644 --- a/federationapi/storage/sqlite3/joined_hosts_table.go +++ b/federationapi/storage/sqlite3/joined_hosts_table.go @@ -90,22 +90,14 @@ func NewSQLiteJoinedHostsTable(db *sql.DB) (s *joinedHostsStatements, err error) if err != nil { return } - if s.insertJoinedHostsStmt, err = db.Prepare(insertJoinedHostsSQL); err != nil { - return - } - if s.deleteJoinedHostsStmt, err = db.Prepare(deleteJoinedHostsSQL); err != nil { - return - } - if s.deleteJoinedHostsForRoomStmt, err = s.db.Prepare(deleteJoinedHostsForRoomSQL); err != nil { - return - } - if s.selectJoinedHostsStmt, err = db.Prepare(selectJoinedHostsSQL); err != nil { - return - } - if s.selectAllJoinedHostsStmt, err = db.Prepare(selectAllJoinedHostsSQL); err != nil { - return - } - return + + return s, sqlutil.StatementList{ + {&s.insertJoinedHostsStmt, insertJoinedHostsSQL}, + {&s.deleteJoinedHostsStmt, deleteJoinedHostsSQL}, + {&s.deleteJoinedHostsForRoomStmt, deleteJoinedHostsForRoomSQL}, + {&s.selectJoinedHostsStmt, selectJoinedHostsSQL}, + {&s.selectAllJoinedHostsStmt, selectAllJoinedHostsSQL}, + }.Prepare(db) } func (s *joinedHostsStatements) InsertJoinedHosts( diff --git a/federationapi/storage/sqlite3/notary_server_keys_json_table.go b/federationapi/storage/sqlite3/notary_server_keys_json_table.go index 4a028fa20b..24875569b8 100644 --- a/federationapi/storage/sqlite3/notary_server_keys_json_table.go +++ b/federationapi/storage/sqlite3/notary_server_keys_json_table.go @@ -19,6 +19,7 @@ import ( "database/sql" "github.com/matrix-org/dendrite/federationapi/storage/tables" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" ) @@ -49,10 +50,9 @@ func NewSQLiteNotaryServerKeysTable(db *sql.DB) (s *notaryServerKeysStatements, return } - if s.insertServerKeysJSONStmt, err = db.Prepare(insertServerKeysJSONSQL); err != nil { - return - } - return + return s, sqlutil.StatementList{ + {&s.insertServerKeysJSONStmt, insertServerKeysJSONSQL}, + }.Prepare(db) } func (s *notaryServerKeysStatements) InsertJSONResponse( diff --git a/federationapi/storage/sqlite3/notary_server_keys_metadata_table.go b/federationapi/storage/sqlite3/notary_server_keys_metadata_table.go index 55709a9623..7179eb8d6f 100644 --- a/federationapi/storage/sqlite3/notary_server_keys_metadata_table.go +++ b/federationapi/storage/sqlite3/notary_server_keys_metadata_table.go @@ -92,19 +92,12 @@ func NewSQLiteNotaryServerKeysMetadataTable(db *sql.DB) (s *notaryServerKeysMeta return } - if s.upsertServerKeysStmt, err = db.Prepare(upsertServerKeysSQL); err != nil { - return - } - if s.selectNotaryKeyResponsesStmt, err = db.Prepare(selectNotaryKeyResponsesSQL); err != nil { - return - } - if s.selectNotaryKeyMetadataStmt, err = db.Prepare(selectNotaryKeyMetadataSQL); err != nil { - return - } - if s.deleteUnusedServerKeysJSONStmt, err = db.Prepare(deleteUnusedServerKeysJSONSQL); err != nil { - return - } - return + return s, sqlutil.StatementList{ + {&s.upsertServerKeysStmt, upsertServerKeysSQL}, + {&s.selectNotaryKeyResponsesStmt, selectNotaryKeyResponsesSQL}, + {&s.selectNotaryKeyMetadataStmt, selectNotaryKeyMetadataSQL}, + {&s.deleteUnusedServerKeysJSONStmt, deleteUnusedServerKeysJSONSQL}, + }.Prepare(db) } func (s *notaryServerKeysMetadataStatements) UpsertKey( diff --git a/federationapi/storage/sqlite3/queue_json_table.go b/federationapi/storage/sqlite3/queue_json_table.go index fe5896c7f9..0e2806d56d 100644 --- a/federationapi/storage/sqlite3/queue_json_table.go +++ b/federationapi/storage/sqlite3/queue_json_table.go @@ -66,10 +66,10 @@ func NewSQLiteQueueJSONTable(db *sql.DB) (s *queueJSONStatements, err error) { if err != nil { return } - if s.insertJSONStmt, err = db.Prepare(insertJSONSQL); err != nil { - return - } - return + + return s, sqlutil.StatementList{ + {&s.insertJSONStmt, insertJSONSQL}, + }.Prepare(db) } func (s *queueJSONStatements) InsertQueueJSON( diff --git a/federationapi/storage/sqlite3/queue_pdus_table.go b/federationapi/storage/sqlite3/queue_pdus_table.go index aee8b03d60..d8d99f0c08 100644 --- a/federationapi/storage/sqlite3/queue_pdus_table.go +++ b/federationapi/storage/sqlite3/queue_pdus_table.go @@ -87,25 +87,13 @@ func NewSQLiteQueuePDUsTable(db *sql.DB) (s *queuePDUsStatements, err error) { if err != nil { return } - if s.insertQueuePDUStmt, err = db.Prepare(insertQueuePDUSQL); err != nil { - return - } - //if s.deleteQueuePDUsStmt, err = db.Prepare(deleteQueuePDUsSQL); err != nil { - // return - //} - if s.selectQueueNextTransactionIDStmt, err = db.Prepare(selectQueueNextTransactionIDSQL); err != nil { - return - } - if s.selectQueuePDUsStmt, err = db.Prepare(selectQueuePDUsSQL); err != nil { - return - } - if s.selectQueueReferenceJSONCountStmt, err = db.Prepare(selectQueuePDUsReferenceJSONCountSQL); err != nil { - return - } - if s.selectQueueServerNamesStmt, err = db.Prepare(selectQueuePDUsServerNamesSQL); err != nil { - return - } - return + return s, sqlutil.StatementList{ + {&s.insertQueuePDUStmt, insertQueuePDUSQL}, + {&s.selectQueueNextTransactionIDStmt, selectQueueNextTransactionIDSQL}, + {&s.selectQueuePDUsStmt, selectQueuePDUsSQL}, + {&s.selectQueueReferenceJSONCountStmt, selectQueuePDUsReferenceJSONCountSQL}, + {&s.selectQueueServerNamesStmt, selectQueuePDUsServerNamesSQL}, + }.Prepare(db) } func (s *queuePDUsStatements) InsertQueuePDU( diff --git a/federationapi/storage/sqlite3/server_key_table.go b/federationapi/storage/sqlite3/server_key_table.go index 9b89649f60..b32ff0926a 100644 --- a/federationapi/storage/sqlite3/server_key_table.go +++ b/federationapi/storage/sqlite3/server_key_table.go @@ -74,13 +74,10 @@ func NewSQLiteServerSigningKeysTable(db *sql.DB) (s *serverSigningKeyStatements, if err != nil { return } - if s.bulkSelectServerKeysStmt, err = db.Prepare(bulkSelectServerSigningKeysSQL); err != nil { - return - } - if s.upsertServerKeysStmt, err = db.Prepare(upsertServerSigningKeysSQL); err != nil { - return - } - return s, nil + return s, sqlutil.StatementList{ + {&s.bulkSelectServerKeysStmt, bulkSelectServerSigningKeysSQL}, + {&s.upsertServerKeysStmt, upsertServerSigningKeysSQL}, + }.Prepare(db) } func (s *serverSigningKeyStatements) BulkSelectServerKeys( diff --git a/go.mod b/go.mod index cd9e0838fa..7ac294f2a1 100644 --- a/go.mod +++ b/go.mod @@ -30,7 +30,6 @@ require ( github.com/nats-io/nats.go v1.24.0 github.com/neilalexander/utp v0.1.1-0.20210727203401-54ae7b1cd5f9 github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 - github.com/ngrok/sqlmw v0.0.0-20220520173518-97c9c04efc79 github.com/opentracing/opentracing-go v1.2.0 github.com/patrickmn/go-cache v2.1.0+incompatible github.com/pkg/errors v0.9.1 diff --git a/go.sum b/go.sum index 1484f8aef2..bf8fbdf3db 100644 --- a/go.sum +++ b/go.sum @@ -368,8 +368,6 @@ github.com/neilalexander/utp v0.1.1-0.20210727203401-54ae7b1cd5f9 h1:lrVQzBtkeQE github.com/neilalexander/utp v0.1.1-0.20210727203401-54ae7b1cd5f9/go.mod h1:NPHGhPc0/wudcaCqL/H5AOddkRf8GPRhzOujuUKGQu8= github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ= github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8= -github.com/ngrok/sqlmw v0.0.0-20220520173518-97c9c04efc79 h1:Dmx8g2747UTVPzSkmohk84S3g/uWqd6+f4SSLPhLcfA= -github.com/ngrok/sqlmw v0.0.0-20220520173518-97c9c04efc79/go.mod h1:E26fwEtRNigBfFfHDWsklmo0T7Ixbg0XXgck+Hq4O9k= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/onsi/ginkgo/v2 v2.3.0 h1:kUMoxMoQG3ogk/QWyKh3zibV7BKZ+xBpWil1cTylVqc= github.com/onsi/ginkgo/v2 v2.3.0/go.mod h1:Eew0uilEqZmIEZr8JrvYlvOM7Rr6xzTmMV8AyFNU9d0= diff --git a/internal/sqlutil/sqlite_cgo.go b/internal/sqlutil/sqlite_cgo.go index efb743fc7a..2fe2396ff3 100644 --- a/internal/sqlutil/sqlite_cgo.go +++ b/internal/sqlutil/sqlite_cgo.go @@ -4,7 +4,6 @@ package sqlutil import ( - "github.com/mattn/go-sqlite3" _ "github.com/mattn/go-sqlite3" ) @@ -13,7 +12,3 @@ const SQLITE_DRIVER_NAME = "sqlite3" func sqliteDSNExtension(dsn string) string { return dsn } - -func sqliteDriver() *sqlite3.SQLiteDriver { - return &sqlite3.SQLiteDriver{} -} diff --git a/internal/sqlutil/sqlite_native.go b/internal/sqlutil/sqlite_native.go index ed500afc61..3c545b659f 100644 --- a/internal/sqlutil/sqlite_native.go +++ b/internal/sqlutil/sqlite_native.go @@ -4,7 +4,6 @@ package sqlutil import ( - "modernc.org/sqlite" "strings" ) @@ -23,7 +22,3 @@ func sqliteDSNExtension(dsn string) string { dsn += "_pragma=busy_timeout%3d10000" return dsn } - -func sqliteDriver() *sqlite.Driver { - return &sqlite.Driver{} -} diff --git a/internal/sqlutil/sqlutil.go b/internal/sqlutil/sqlutil.go index 39a067e529..b2d0d2186d 100644 --- a/internal/sqlutil/sqlutil.go +++ b/internal/sqlutil/sqlutil.go @@ -13,8 +13,7 @@ import ( var skipSanityChecks = flag.Bool("skip-db-sanity", false, "Ignore sanity checks on the database connections (NOT RECOMMENDED!)") // Open opens a database specified by its database driver name and a driver-specific data source name, -// usually consisting of at least a database name and connection information. Includes tracing driver -// if DENDRITE_TRACE_SQL=1 +// usually consisting of at least a database name and connection information. func Open(dbProperties *config.DatabaseOptions, writer Writer) (*sql.DB, error) { var err error var driverName, dsn string @@ -32,10 +31,6 @@ func Open(dbProperties *config.DatabaseOptions, writer Writer) (*sql.DB, error) default: return nil, fmt.Errorf("invalid database connection string %q", dbProperties.ConnectionString) } - if tracingEnabled { - // install the wrapped driver - driverName += "-trace" - } db, err := sql.Open(driverName, dsn) if err != nil { return nil, err diff --git a/internal/sqlutil/trace.go b/internal/sqlutil/trace.go deleted file mode 100644 index 7b637106ba..0000000000 --- a/internal/sqlutil/trace.go +++ /dev/null @@ -1,109 +0,0 @@ -// Copyright 2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package sqlutil - -import ( - "context" - "database/sql/driver" - "fmt" - "io" - "os" - "runtime" - "strconv" - "strings" - "sync" - "time" - - "github.com/ngrok/sqlmw" - "github.com/sirupsen/logrus" -) - -var tracingEnabled = os.Getenv("DENDRITE_TRACE_SQL") == "1" -var goidToWriter sync.Map - -type traceInterceptor struct { - sqlmw.NullInterceptor -} - -func (in *traceInterceptor) StmtQueryContext(ctx context.Context, stmt driver.StmtQueryContext, query string, args []driver.NamedValue) (context.Context, driver.Rows, error) { - startedAt := time.Now() - rows, err := stmt.QueryContext(ctx, args) - - trackGoID(query) - - logrus.WithField("duration", time.Since(startedAt)).WithField(logrus.ErrorKey, err).Debug("executed sql query ", query, " args: ", args) - - return ctx, rows, err -} - -func (in *traceInterceptor) StmtExecContext(ctx context.Context, stmt driver.StmtExecContext, query string, args []driver.NamedValue) (driver.Result, error) { - startedAt := time.Now() - result, err := stmt.ExecContext(ctx, args) - - trackGoID(query) - - logrus.WithField("duration", time.Since(startedAt)).WithField(logrus.ErrorKey, err).Debug("executed sql query ", query, " args: ", args) - - return result, err -} - -func (in *traceInterceptor) RowsNext(c context.Context, rows driver.Rows, dest []driver.Value) error { - err := rows.Next(dest) - if err == io.EOF { - // For all cases, we call Next() n+1 times, the first to populate the initial dest, then eventually - // it will io.EOF. If we log on each Next() call we log the last element twice, so don't. - return err - } - cols := rows.Columns() - logrus.Debug(strings.Join(cols, " | ")) - - b := strings.Builder{} - for i, val := range dest { - b.WriteString(fmt.Sprintf("%q", val)) - if i+1 <= len(dest)-1 { - b.WriteString(" | ") - } - } - logrus.Debug(b.String()) - return err -} - -func trackGoID(query string) { - thisGoID := goid() - if _, ok := goidToWriter.Load(thisGoID); ok { - return // we're on a writer goroutine - } - - q := strings.TrimSpace(query) - if strings.HasPrefix(q, "SELECT") { - return // SELECTs can go on other goroutines - } - logrus.Warnf("unsafe goid %d: SQL executed not on an ExclusiveWriter: %s", thisGoID, q) -} - -func init() { - registerDrivers() -} - -func goid() int { - var buf [64]byte - n := runtime.Stack(buf[:], false) - idField := strings.Fields(strings.TrimPrefix(string(buf[:n]), "goroutine "))[0] - id, err := strconv.Atoi(idField) - if err != nil { - panic(fmt.Sprintf("cannot get goroutine id: %v", err)) - } - return id -} diff --git a/internal/sqlutil/trace_driver.go b/internal/sqlutil/trace_driver.go deleted file mode 100644 index a2e0d12e20..0000000000 --- a/internal/sqlutil/trace_driver.go +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright 2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//go:build !wasm -// +build !wasm - -package sqlutil - -import ( - "database/sql" - - "github.com/lib/pq" - "github.com/ngrok/sqlmw" -) - -func registerDrivers() { - if !tracingEnabled { - return - } - // install the wrapped drivers - sql.Register("postgres-trace", sqlmw.Driver(&pq.Driver{}, new(traceInterceptor))) - sql.Register("sqlite3-trace", sqlmw.Driver(sqliteDriver(), new(traceInterceptor))) - -} diff --git a/internal/sqlutil/trace_driver_wasm.go b/internal/sqlutil/trace_driver_wasm.go deleted file mode 100644 index 51b60c3c8c..0000000000 --- a/internal/sqlutil/trace_driver_wasm.go +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright 2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//go:build wasm -// +build wasm - -package sqlutil - -import ( - "database/sql" - - sqlitejs "github.com/matrix-org/go-sqlite3-js" - "github.com/ngrok/sqlmw" -) - -func registerDrivers() { - if !tracingEnabled { - return - } - // install the wrapped drivers - sql.Register("sqlite3_js-trace", sqlmw.Driver(&sqlitejs.SqliteJsDriver{}, new(traceInterceptor))) - -} diff --git a/internal/sqlutil/writer_exclusive.go b/internal/sqlutil/writer_exclusive.go index 8eff3ce550..c6a271c1c0 100644 --- a/internal/sqlutil/writer_exclusive.go +++ b/internal/sqlutil/writer_exclusive.go @@ -60,11 +60,6 @@ func (w *ExclusiveWriter) run() { if !w.running.CompareAndSwap(false, true) { return } - if tracingEnabled { - gid := goid() - goidToWriter.Store(gid, w) - defer goidToWriter.Delete(gid) - } defer w.running.Store(false) for task := range w.todo { diff --git a/syncapi/storage/postgres/account_data_table.go b/syncapi/storage/postgres/account_data_table.go index aa54cb08fb..8147b48689 100644 --- a/syncapi/storage/postgres/account_data_table.go +++ b/syncapi/storage/postgres/account_data_table.go @@ -78,16 +78,11 @@ func NewPostgresAccountDataTable(db *sql.DB) (tables.AccountData, error) { if err != nil { return nil, err } - if s.insertAccountDataStmt, err = db.Prepare(insertAccountDataSQL); err != nil { - return nil, err - } - if s.selectAccountDataInRangeStmt, err = db.Prepare(selectAccountDataInRangeSQL); err != nil { - return nil, err - } - if s.selectMaxAccountDataIDStmt, err = db.Prepare(selectMaxAccountDataIDSQL); err != nil { - return nil, err - } - return s, nil + return s, sqlutil.StatementList{ + {&s.insertAccountDataStmt, insertAccountDataSQL}, + {&s.selectAccountDataInRangeStmt, selectAccountDataInRangeSQL}, + {&s.selectMaxAccountDataIDStmt, selectMaxAccountDataIDSQL}, + }.Prepare(db) } func (s *accountDataStatements) InsertAccountData( diff --git a/syncapi/storage/postgres/filter_table.go b/syncapi/storage/postgres/filter_table.go index 86cec36257..44c3de72ef 100644 --- a/syncapi/storage/postgres/filter_table.go +++ b/syncapi/storage/postgres/filter_table.go @@ -61,16 +61,11 @@ func NewPostgresFilterTable(db *sql.DB) (tables.Filter, error) { return nil, err } s := &filterStatements{} - if s.selectFilterStmt, err = db.Prepare(selectFilterSQL); err != nil { - return nil, err - } - if s.selectFilterIDByContentStmt, err = db.Prepare(selectFilterIDByContentSQL); err != nil { - return nil, err - } - if s.insertFilterStmt, err = db.Prepare(insertFilterSQL); err != nil { - return nil, err - } - return s, nil + return s, sqlutil.StatementList{ + {&s.selectFilterStmt, selectFilterSQL}, + {&s.selectFilterIDByContentStmt, selectFilterIDByContentSQL}, + {&s.insertFilterStmt, insertFilterSQL}, + }.Prepare(db) } func (s *filterStatements) SelectFilter( diff --git a/syncapi/storage/postgres/ignores_table.go b/syncapi/storage/postgres/ignores_table.go index 97660725c1..a511c747cc 100644 --- a/syncapi/storage/postgres/ignores_table.go +++ b/syncapi/storage/postgres/ignores_table.go @@ -52,13 +52,11 @@ func NewPostgresIgnoresTable(db *sql.DB) (tables.Ignores, error) { return nil, err } s := &ignoresStatements{} - if s.selectIgnoresStmt, err = db.Prepare(selectIgnoresSQL); err != nil { - return nil, err - } - if s.upsertIgnoresStmt, err = db.Prepare(upsertIgnoresSQL); err != nil { - return nil, err - } - return s, nil + + return s, sqlutil.StatementList{ + {&s.selectIgnoresStmt, selectIgnoresSQL}, + {&s.upsertIgnoresStmt, upsertIgnoresSQL}, + }.Prepare(db) } func (s *ignoresStatements) SelectIgnores( diff --git a/syncapi/storage/postgres/send_to_device_table.go b/syncapi/storage/postgres/send_to_device_table.go index 6ab1f0f48b..88b86ef7b4 100644 --- a/syncapi/storage/postgres/send_to_device_table.go +++ b/syncapi/storage/postgres/send_to_device_table.go @@ -88,19 +88,12 @@ func NewPostgresSendToDeviceTable(db *sql.DB) (tables.SendToDevice, error) { if err != nil { return nil, err } - if s.insertSendToDeviceMessageStmt, err = db.Prepare(insertSendToDeviceMessageSQL); err != nil { - return nil, err - } - if s.selectSendToDeviceMessagesStmt, err = db.Prepare(selectSendToDeviceMessagesSQL); err != nil { - return nil, err - } - if s.deleteSendToDeviceMessagesStmt, err = db.Prepare(deleteSendToDeviceMessagesSQL); err != nil { - return nil, err - } - if s.selectMaxSendToDeviceIDStmt, err = db.Prepare(selectMaxSendToDeviceIDSQL); err != nil { - return nil, err - } - return s, nil + return s, sqlutil.StatementList{ + {&s.insertSendToDeviceMessageStmt, insertSendToDeviceMessageSQL}, + {&s.selectSendToDeviceMessagesStmt, selectSendToDeviceMessagesSQL}, + {&s.deleteSendToDeviceMessagesStmt, deleteSendToDeviceMessagesSQL}, + {&s.selectMaxSendToDeviceIDStmt, selectMaxSendToDeviceIDSQL}, + }.Prepare(db) } func (s *sendToDeviceStatements) InsertSendToDeviceMessage( diff --git a/syncapi/storage/sqlite3/account_data_table.go b/syncapi/storage/sqlite3/account_data_table.go index de0e72dbdc..eaf5223409 100644 --- a/syncapi/storage/sqlite3/account_data_table.go +++ b/syncapi/storage/sqlite3/account_data_table.go @@ -66,16 +66,11 @@ func NewSqliteAccountDataTable(db *sql.DB, streamID *StreamIDStatements) (tables if err != nil { return nil, err } - if s.insertAccountDataStmt, err = db.Prepare(insertAccountDataSQL); err != nil { - return nil, err - } - if s.selectMaxAccountDataIDStmt, err = db.Prepare(selectMaxAccountDataIDSQL); err != nil { - return nil, err - } - if s.selectAccountDataInRangeStmt, err = db.Prepare(selectAccountDataInRangeSQL); err != nil { - return nil, err - } - return s, nil + return s, sqlutil.StatementList{ + {&s.insertAccountDataStmt, insertAccountDataSQL}, + {&s.selectMaxAccountDataIDStmt, selectMaxAccountDataIDSQL}, + {&s.selectAccountDataInRangeStmt, selectAccountDataInRangeSQL}, + }.Prepare(db) } func (s *accountDataStatements) InsertAccountData( diff --git a/syncapi/storage/sqlite3/filter_table.go b/syncapi/storage/sqlite3/filter_table.go index 5f1e980eb0..0132a7664a 100644 --- a/syncapi/storage/sqlite3/filter_table.go +++ b/syncapi/storage/sqlite3/filter_table.go @@ -65,16 +65,11 @@ func NewSqliteFilterTable(db *sql.DB) (tables.Filter, error) { s := &filterStatements{ db: db, } - if s.selectFilterStmt, err = db.Prepare(selectFilterSQL); err != nil { - return nil, err - } - if s.selectFilterIDByContentStmt, err = db.Prepare(selectFilterIDByContentSQL); err != nil { - return nil, err - } - if s.insertFilterStmt, err = db.Prepare(insertFilterSQL); err != nil { - return nil, err - } - return s, nil + return s, sqlutil.StatementList{ + {&s.selectFilterStmt, selectFilterSQL}, + {&s.selectFilterIDByContentStmt, selectFilterIDByContentSQL}, + {&s.insertFilterStmt, insertFilterSQL}, + }.Prepare(db) } func (s *filterStatements) SelectFilter( diff --git a/syncapi/storage/sqlite3/ignores_table.go b/syncapi/storage/sqlite3/ignores_table.go index 5ee1a9fa00..bff5780b06 100644 --- a/syncapi/storage/sqlite3/ignores_table.go +++ b/syncapi/storage/sqlite3/ignores_table.go @@ -52,13 +52,10 @@ func NewSqliteIgnoresTable(db *sql.DB) (tables.Ignores, error) { return nil, err } s := &ignoresStatements{} - if s.selectIgnoresStmt, err = db.Prepare(selectIgnoresSQL); err != nil { - return nil, err - } - if s.upsertIgnoresStmt, err = db.Prepare(upsertIgnoresSQL); err != nil { - return nil, err - } - return s, nil + return s, sqlutil.StatementList{ + {&s.selectIgnoresStmt, selectIgnoresSQL}, + {&s.upsertIgnoresStmt, upsertIgnoresSQL}, + }.Prepare(db) } func (s *ignoresStatements) SelectIgnores( diff --git a/syncapi/storage/sqlite3/send_to_device_table.go b/syncapi/storage/sqlite3/send_to_device_table.go index 0da06506c3..998a063cd4 100644 --- a/syncapi/storage/sqlite3/send_to_device_table.go +++ b/syncapi/storage/sqlite3/send_to_device_table.go @@ -88,19 +88,12 @@ func NewSqliteSendToDeviceTable(db *sql.DB) (tables.SendToDevice, error) { if err != nil { return nil, err } - if s.insertSendToDeviceMessageStmt, err = db.Prepare(insertSendToDeviceMessageSQL); err != nil { - return nil, err - } - if s.selectSendToDeviceMessagesStmt, err = db.Prepare(selectSendToDeviceMessagesSQL); err != nil { - return nil, err - } - if s.deleteSendToDeviceMessagesStmt, err = db.Prepare(deleteSendToDeviceMessagesSQL); err != nil { - return nil, err - } - if s.selectMaxSendToDeviceIDStmt, err = db.Prepare(selectMaxSendToDeviceIDSQL); err != nil { - return nil, err - } - return s, nil + return s, sqlutil.StatementList{ + {&s.insertSendToDeviceMessageStmt, insertSendToDeviceMessageSQL}, + {&s.selectSendToDeviceMessagesStmt, selectSendToDeviceMessagesSQL}, + {&s.deleteSendToDeviceMessagesStmt, deleteSendToDeviceMessagesSQL}, + {&s.selectMaxSendToDeviceIDStmt, selectMaxSendToDeviceIDSQL}, + }.Prepare(db) } func (s *sendToDeviceStatements) InsertSendToDeviceMessage( diff --git a/syncapi/storage/sqlite3/stream_id_table.go b/syncapi/storage/sqlite3/stream_id_table.go index a4bba508ec..b51eccf553 100644 --- a/syncapi/storage/sqlite3/stream_id_table.go +++ b/syncapi/storage/sqlite3/stream_id_table.go @@ -47,10 +47,9 @@ func (s *StreamIDStatements) Prepare(db *sql.DB) (err error) { if err != nil { return } - if s.increaseStreamIDStmt, err = db.Prepare(increaseStreamIDStmt); err != nil { - return - } - return + return sqlutil.StatementList{ + {&s.increaseStreamIDStmt, increaseStreamIDStmt}, + }.Prepare(db) } func (s *StreamIDStatements) nextPDUID(ctx context.Context, txn *sql.Tx) (pos types.StreamPosition, err error) { From 05f72fc4be6755446d084c0f679fdafbb5fda975 Mon Sep 17 00:00:00 2001 From: Alex Kirk Date: Mon, 27 Mar 2023 10:55:36 +0200 Subject: [PATCH 13/42] Update docs and sample config for the relay_api (#3011) This adds an empty `relay_api` section to the sample configuration. For SQLite environments, or others where a `database.connection_string` is needed for each section, there should be an entry in the configuration sample as a basis. This PR also changes the "Configuring Dendrite" documentation in that respect. The requirement was introduced in #2917. When upgrading dendrite, it will complain about `relay_api.database.connection_string` not being configured. ### Pull Request Checklist * [x] I have added Go unit tests or [Complement integration tests](https://github.com/matrix-org/complement) for this PR _or_ I have justified why this PR doesn't need tests * [x] Pull request includes a [sign off below using a legally identifiable name](https://matrix-org.github.io/dendrite/development/contributing#sign-off) _or_ I have already signed off privately Signed-off-by: `Alex Kirk ` --- docs/installation/7_configuration.md | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/docs/installation/7_configuration.md b/docs/installation/7_configuration.md index 5f123bfca5..0cc67b1561 100644 --- a/docs/installation/7_configuration.md +++ b/docs/installation/7_configuration.md @@ -10,7 +10,7 @@ permalink: /installation/configuration A YAML configuration file is used to configure Dendrite. A sample configuration file is present in the top level of the Dendrite repository: -* [`dendrite-sample.monolith.yaml`](https://github.com/matrix-org/dendrite/blob/main/dendrite-sample.monolith.yaml) +* [`dendrite-sample.yaml`](https://github.com/matrix-org/dendrite/blob/main/dendrite-sample.yaml) You will need to duplicate the sample, calling it `dendrite.yaml` for example, and then tailor it to your installation. At a minimum, you will need to populate the following @@ -86,9 +86,8 @@ that you chose. ### Global connection pool -If you are running a monolith deployment and want to use a single connection pool to a -single PostgreSQL database, then you must uncomment and configure the `database` section -within the `global` section: +If you want to use a single connection pool to a single PostgreSQL database, then you must +uncomment and configure the `database` section within the `global` section: ```yaml global: @@ -102,15 +101,15 @@ global: **You must then remove or comment out** the `database` sections from other areas of the configuration file, e.g. under the `app_service_api`, `federation_api`, `key_server`, -`media_api`, `mscs`, `room_server`, `sync_api` and `user_api` blocks, otherwise these will -override the `global` database configuration. +`media_api`, `mscs`, `relay_api`, `room_server`, `sync_api` and `user_api` blocks, otherwise +these will override the `global` database configuration. ### Per-component connections (all other configurations) If you are are using SQLite databases or separate PostgreSQL databases per component, then you must instead configure the `database` sections under each of the component blocks ,e.g. under the `app_service_api`, `federation_api`, `key_server`, -`media_api`, `mscs`, `room_server`, `sync_api` and `user_api` blocks. +`media_api`, `mscs`, `relay_api`, `room_server`, `sync_api` and `user_api` blocks. For example, with PostgreSQL: From e2d2482ca67e0a87f3726027093b22480dc1af3f Mon Sep 17 00:00:00 2001 From: Till Faelligen <2353100+S7evinK@users.noreply.github.com> Date: Mon, 27 Mar 2023 11:07:30 +0200 Subject: [PATCH 14/42] Get the logs for dendrite when installing the chart --- .github/workflows/k8s.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/k8s.yml b/.github/workflows/k8s.yml index fc5e8c9069..af2750356c 100644 --- a/.github/workflows/k8s.yml +++ b/.github/workflows/k8s.yml @@ -84,6 +84,7 @@ jobs: kubectl get pods -A kubectl get services kubectl get ingress + kubectl logs -l app.kubernetes.io/name=dendrite - name: Run create account run: | podName=$(kubectl get pods -l app.kubernetes.io/name=dendrite -o name) From aa1bda4c58d20e7d14267f9c87fab8efd7ae36ad Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Mon, 27 Mar 2023 11:26:26 +0200 Subject: [PATCH 15/42] Add AS invite test, fix issue with invitations being processed twice (#3020) The AS roomserver consumer would receive the events twice, one time as type `OutputTypeNewInviteEvent` and the other time as `OutputTypeNewRoomEvent`. [skip ci] --- appservice/appservice_test.go | 85 ++++++++++++++++++++++++++++++ appservice/consumers/roomserver.go | 6 --- 2 files changed, 85 insertions(+), 6 deletions(-) diff --git a/appservice/appservice_test.go b/appservice/appservice_test.go index 6c8a07b5c6..752901a9c5 100644 --- a/appservice/appservice_test.go +++ b/appservice/appservice_test.go @@ -16,10 +16,12 @@ import ( "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver" + rsapi "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/jetstream" "github.com/matrix-org/dendrite/test" "github.com/matrix-org/dendrite/userapi" + "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/dendrite/test/testrig" ) @@ -212,3 +214,86 @@ func testProtocol(t *testing.T, asAPI api.AppServiceInternalAPI, proto string, w t.Errorf("unexpected result for Protocols(%s): %+v, expected %+v", proto, protoResp.Protocols[proto], wantResult) } } + +// Tests that the roomserver consumer only receives one invite +func TestRoomserverConsumerOneInvite(t *testing.T) { + + alice := test.NewUser(t) + bob := test.NewUser(t) + room := test.NewRoom(t, alice) + + // Invite Bob + room.CreateAndInsert(t, alice, gomatrixserverlib.MRoomMember, map[string]interface{}{ + "membership": "invite", + }, test.WithStateKey(bob.ID)) + + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + cfg, processCtx, closeDB := testrig.CreateConfig(t, dbType) + defer closeDB() + cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) + natsInstance := &jetstream.NATSInstance{} + + evChan := make(chan struct{}) + // create a dummy AS url, handling the events + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var txn gomatrixserverlib.ApplicationServiceTransaction + err := json.NewDecoder(r.Body).Decode(&txn) + if err != nil { + t.Fatal(err) + } + for _, ev := range txn.Events { + if ev.Type != gomatrixserverlib.MRoomMember { + continue + } + // Usually we would check the event content for the membership, but since + // we only invited bob, this should be fine for this test. + if ev.StateKey != nil && *ev.StateKey == bob.ID { + evChan <- struct{}{} + } + } + })) + defer srv.Close() + + // Create a dummy application service + cfg.AppServiceAPI.Derived.ApplicationServices = []config.ApplicationService{ + { + ID: "someID", + URL: srv.URL, + ASToken: "", + HSToken: "", + SenderLocalpart: "senderLocalPart", + NamespaceMap: map[string][]config.ApplicationServiceNamespace{ + "users": {{RegexpObject: regexp.MustCompile(bob.ID)}}, + "aliases": {{RegexpObject: regexp.MustCompile(room.ID)}}, + }, + }, + } + + caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) + // Create required internal APIs + rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, natsInstance, caches, caching.DisableMetrics) + rsAPI.SetFederationAPI(nil, nil) + usrAPI := userapi.NewInternalAPI(processCtx, cfg, cm, natsInstance, rsAPI, nil) + // start the consumer + appservice.NewInternalAPI(processCtx, cfg, natsInstance, usrAPI, rsAPI) + + // Create the room + if err := rsapi.SendEvents(context.Background(), rsAPI, rsapi.KindNew, room.Events(), "test", "test", "test", nil, false); err != nil { + t.Fatalf("failed to send events: %v", err) + } + var seenInvitesForBob int + waitLoop: + for { + select { + case <-time.After(time.Millisecond * 50): // wait for the AS to process the events + break waitLoop + case <-evChan: + seenInvitesForBob++ + if seenInvitesForBob != 1 { + t.Fatalf("received unexpected invites: %d", seenInvitesForBob) + } + } + } + close(evChan) + }) +} diff --git a/appservice/consumers/roomserver.go b/appservice/consumers/roomserver.go index 528de63e88..16b3b82319 100644 --- a/appservice/consumers/roomserver.go +++ b/appservice/consumers/roomserver.go @@ -140,12 +140,6 @@ func (s *OutputRoomEventConsumer) onMessage( } } - case api.OutputTypeNewInviteEvent: - if output.NewInviteEvent == nil || !s.appserviceIsInterestedInEvent(ctx, output.NewInviteEvent.Event, state.ApplicationService) { - continue - } - events = append(events, output.NewInviteEvent.Event) - default: continue } From e8b2162a01bf0e735869d5a2b9be258cb380255e Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Mon, 27 Mar 2023 11:26:52 +0200 Subject: [PATCH 16/42] Add `/search` tests (#3025) --- internal/fulltext/bleve.go | 47 ++++++ internal/fulltext/bleve_test.go | 46 ++++-- internal/fulltext/bleve_wasm.go | 5 + syncapi/routing/search.go | 77 ++++++---- syncapi/routing/search_test.go | 264 ++++++++++++++++++++++++++++++++ 5 files changed, 389 insertions(+), 50 deletions(-) create mode 100644 syncapi/routing/search_test.go diff --git a/internal/fulltext/bleve.go b/internal/fulltext/bleve.go index dea7c504c6..f7412470d8 100644 --- a/internal/fulltext/bleve.go +++ b/internal/fulltext/bleve.go @@ -18,6 +18,7 @@ package fulltext import ( + "regexp" "strings" "github.com/blevesearch/bleve/v2" @@ -60,6 +61,7 @@ type Indexer interface { Index(elements ...IndexElement) error Delete(eventID string) error Search(term string, roomIDs, keys []string, limit, from int, orderByStreamPos bool) (*bleve.SearchResult, error) + GetHighlights(result *bleve.SearchResult) []string Close() error } @@ -124,6 +126,47 @@ func (f *Search) Delete(eventID string) error { return f.FulltextIndex.Delete(eventID) } +var highlightMatcher = regexp.MustCompile("(.*?)") + +// GetHighlights extracts the highlights from a SearchResult. +func (f *Search) GetHighlights(result *bleve.SearchResult) []string { + if result == nil { + return []string{} + } + + seenMatches := make(map[string]struct{}) + + for _, hit := range result.Hits { + if hit.Fragments == nil { + continue + } + fragments, ok := hit.Fragments["Content"] + if !ok { + continue + } + for _, x := range fragments { + substringMatches := highlightMatcher.FindAllStringSubmatch(x, -1) + for _, matches := range substringMatches { + for i := range matches { + if i == 0 { // skip first match, this is the complete substring match + continue + } + if _, ok := seenMatches[matches[i]]; ok { + continue + } + seenMatches[matches[i]] = struct{}{} + } + } + } + } + + res := make([]string, 0, len(seenMatches)) + for m := range seenMatches { + res = append(res, m) + } + return res +} + // Search searches the index given a search term, roomIDs and keys. func (f *Search) Search(term string, roomIDs, keys []string, limit, from int, orderByStreamPos bool) (*bleve.SearchResult, error) { qry := bleve.NewConjunctionQuery() @@ -163,6 +206,10 @@ func (f *Search) Search(term string, roomIDs, keys []string, limit, from int, or s.SortBy([]string{"-StreamPosition"}) } + // Highlight some words + s.Highlight = bleve.NewHighlight() + s.Highlight.Fields = []string{"Content"} + return f.FulltextIndex.Search(s) } diff --git a/internal/fulltext/bleve_test.go b/internal/fulltext/bleve_test.go index bd8289d58b..a77c239372 100644 --- a/internal/fulltext/bleve_test.go +++ b/internal/fulltext/bleve_test.go @@ -160,14 +160,16 @@ func TestSearch(t *testing.T) { roomIndex []int } tests := []struct { - name string - args args - wantCount int - wantErr bool + name string + args args + wantCount int + wantErr bool + wantHighlights []string }{ { - name: "Can search for many results in one room", - wantCount: 16, + name: "Can search for many results in one room", + wantCount: 16, + wantHighlights: []string{"lorem"}, args: args{ term: "lorem", roomIndex: []int{0}, @@ -175,8 +177,9 @@ func TestSearch(t *testing.T) { }, }, { - name: "Can search for one result in one room", - wantCount: 1, + name: "Can search for one result in one room", + wantCount: 1, + wantHighlights: []string{"lorem"}, args: args{ term: "lorem", roomIndex: []int{16}, @@ -184,8 +187,9 @@ func TestSearch(t *testing.T) { }, }, { - name: "Can search for many results in multiple rooms", - wantCount: 17, + name: "Can search for many results in multiple rooms", + wantCount: 17, + wantHighlights: []string{"lorem"}, args: args{ term: "lorem", roomIndex: []int{0, 16}, @@ -193,8 +197,9 @@ func TestSearch(t *testing.T) { }, }, { - name: "Can search for many results in all rooms, reversed", - wantCount: 30, + name: "Can search for many results in all rooms, reversed", + wantCount: 30, + wantHighlights: []string{"lorem"}, args: args{ term: "lorem", limit: 30, @@ -202,8 +207,9 @@ func TestSearch(t *testing.T) { }, }, { - name: "Can search for specific search room name", - wantCount: 1, + name: "Can search for specific search room name", + wantCount: 1, + wantHighlights: []string{"testing"}, args: args{ term: "testing", roomIndex: []int{}, @@ -212,8 +218,9 @@ func TestSearch(t *testing.T) { }, }, { - name: "Can search for specific search room topic", - wantCount: 1, + name: "Can search for specific search room topic", + wantCount: 1, + wantHighlights: []string{"fulltext"}, args: args{ term: "fulltext", roomIndex: []int{}, @@ -222,6 +229,7 @@ func TestSearch(t *testing.T) { }, }, } + for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { f, ctx := mustOpenIndex(t, "") @@ -238,6 +246,12 @@ func TestSearch(t *testing.T) { t.Errorf("Search() error = %v, wantErr %v", err, tt.wantErr) return } + + highlights := f.GetHighlights(got) + if !reflect.DeepEqual(highlights, tt.wantHighlights) { + t.Errorf("Search() got highligts = %v, want %v", highlights, tt.wantHighlights) + } + if !reflect.DeepEqual(len(got.Hits), tt.wantCount) { t.Errorf("Search() got = %v, want %v", len(got.Hits), tt.wantCount) } diff --git a/internal/fulltext/bleve_wasm.go b/internal/fulltext/bleve_wasm.go index 0053ed8c27..12709900b8 100644 --- a/internal/fulltext/bleve_wasm.go +++ b/internal/fulltext/bleve_wasm.go @@ -33,6 +33,7 @@ type Indexer interface { Index(elements ...IndexElement) error Delete(eventID string) error Search(term string, roomIDs, keys []string, limit, from int, orderByStreamPos bool) (SearchResult, error) + GetHighlights(result SearchResult) []string Close() error } @@ -71,3 +72,7 @@ func (f *Search) Delete(eventID string) error { func (f *Search) Search(term string, roomIDs, keys []string, limit, from int, orderByStreamPos bool) (SearchResult, error) { return SearchResult{}, nil } + +func (f *Search) GetHighlights(result SearchResult) []string { + return []string{} +} diff --git a/syncapi/routing/search.go b/syncapi/routing/search.go index 13625b9cbe..69fa52942f 100644 --- a/syncapi/routing/search.go +++ b/syncapi/routing/search.go @@ -19,7 +19,6 @@ import ( "net/http" "sort" "strconv" - "strings" "time" "github.com/blevesearch/bleve/v2/search" @@ -123,8 +122,8 @@ func Search(req *http.Request, device *api.Device, syncDB storage.Database, fts return util.JSONResponse{ Code: http.StatusOK, JSON: SearchResponse{ - SearchCategories: SearchCategories{ - RoomEvents: RoomEvents{ + SearchCategories: SearchCategoriesResponse{ + RoomEvents: RoomEventsResponse{ Count: int(result.Total), NextBatch: nil, }, @@ -158,7 +157,7 @@ func Search(req *http.Request, device *api.Device, syncDB storage.Database, fts } groups := make(map[string]RoomResult) - knownUsersProfiles := make(map[string]ProfileInfo) + knownUsersProfiles := make(map[string]ProfileInfoResponse) // Sort the events by depth, as the returned values aren't ordered if orderByTime { @@ -180,7 +179,7 @@ func Search(req *http.Request, device *api.Device, syncDB storage.Database, fts return jsonerror.InternalServerError() } - profileInfos := make(map[string]ProfileInfo) + profileInfos := make(map[string]ProfileInfoResponse) for _, ev := range append(eventsBefore, eventsAfter...) { profile, ok := knownUsersProfiles[event.Sender()] if !ok { @@ -192,7 +191,7 @@ func Search(req *http.Request, device *api.Device, syncDB storage.Database, fts if stateEvent == nil { continue } - profile = ProfileInfo{ + profile = ProfileInfoResponse{ AvatarURL: gjson.GetBytes(stateEvent.Content(), "avatar_url").Str, DisplayName: gjson.GetBytes(stateEvent.Content(), "displayname").Str, } @@ -237,13 +236,13 @@ func Search(req *http.Request, device *api.Device, syncDB storage.Database, fts } res := SearchResponse{ - SearchCategories: SearchCategories{ - RoomEvents: RoomEvents{ + SearchCategories: SearchCategoriesResponse{ + RoomEvents: RoomEventsResponse{ Count: int(result.Total), Groups: Groups{RoomID: groups}, Results: results, NextBatch: nextBatchResult, - Highlights: strings.Split(searchReq.SearchCategories.RoomEvents.SearchTerm, " "), + Highlights: fts.GetHighlights(result), State: stateForRooms, }, }, @@ -286,30 +285,40 @@ func contextEvents( return eventsBefore, eventsAfter, err } +type EventContext struct { + AfterLimit int `json:"after_limit,omitempty"` + BeforeLimit int `json:"before_limit,omitempty"` + IncludeProfile bool `json:"include_profile,omitempty"` +} + +type GroupBy struct { + Key string `json:"key"` +} + +type Groupings struct { + GroupBy []GroupBy `json:"group_by"` +} + +type RoomEvents struct { + EventContext EventContext `json:"event_context"` + Filter gomatrixserverlib.RoomEventFilter `json:"filter"` + Groupings Groupings `json:"groupings"` + IncludeState bool `json:"include_state"` + Keys []string `json:"keys"` + OrderBy string `json:"order_by"` + SearchTerm string `json:"search_term"` +} + +type SearchCategories struct { + RoomEvents RoomEvents `json:"room_events"` +} + type SearchRequest struct { - SearchCategories struct { - RoomEvents struct { - EventContext struct { - AfterLimit int `json:"after_limit,omitempty"` - BeforeLimit int `json:"before_limit,omitempty"` - IncludeProfile bool `json:"include_profile,omitempty"` - } `json:"event_context"` - Filter gomatrixserverlib.RoomEventFilter `json:"filter"` - Groupings struct { - GroupBy []struct { - Key string `json:"key"` - } `json:"group_by"` - } `json:"groupings"` - IncludeState bool `json:"include_state"` - Keys []string `json:"keys"` - OrderBy string `json:"order_by"` - SearchTerm string `json:"search_term"` - } `json:"room_events"` - } `json:"search_categories"` + SearchCategories SearchCategories `json:"search_categories"` } type SearchResponse struct { - SearchCategories SearchCategories `json:"search_categories"` + SearchCategories SearchCategoriesResponse `json:"search_categories"` } type RoomResult struct { NextBatch *string `json:"next_batch,omitempty"` @@ -332,15 +341,15 @@ type SearchContextResponse struct { EventsAfter []gomatrixserverlib.ClientEvent `json:"events_after"` EventsBefore []gomatrixserverlib.ClientEvent `json:"events_before"` Start string `json:"start"` - ProfileInfo map[string]ProfileInfo `json:"profile_info"` + ProfileInfo map[string]ProfileInfoResponse `json:"profile_info"` } -type ProfileInfo struct { +type ProfileInfoResponse struct { AvatarURL string `json:"avatar_url"` DisplayName string `json:"display_name"` } -type RoomEvents struct { +type RoomEventsResponse struct { Count int `json:"count"` Groups Groups `json:"groups"` Highlights []string `json:"highlights"` @@ -348,6 +357,6 @@ type RoomEvents struct { Results []Result `json:"results"` State map[string][]gomatrixserverlib.ClientEvent `json:"state,omitempty"` } -type SearchCategories struct { - RoomEvents RoomEvents `json:"room_events"` +type SearchCategoriesResponse struct { + RoomEvents RoomEventsResponse `json:"room_events"` } diff --git a/syncapi/routing/search_test.go b/syncapi/routing/search_test.go new file mode 100644 index 0000000000..05479300e9 --- /dev/null +++ b/syncapi/routing/search_test.go @@ -0,0 +1,264 @@ +package routing + +import ( + "bytes" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/matrix-org/dendrite/internal/fulltext" + "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/syncapi/storage" + "github.com/matrix-org/dendrite/syncapi/types" + "github.com/matrix-org/dendrite/test" + "github.com/matrix-org/dendrite/test/testrig" + userapi "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/gomatrixserverlib" + "github.com/stretchr/testify/assert" +) + +func TestSearch(t *testing.T) { + alice := test.NewUser(t) + aliceDevice := userapi.Device{UserID: alice.ID} + room := test.NewRoom(t, alice) + room.CreateAndInsert(t, alice, "m.room.message", map[string]interface{}{"body": "context before"}) + room.CreateAndInsert(t, alice, "m.room.message", map[string]interface{}{"body": "hello world3!"}) + room.CreateAndInsert(t, alice, "m.room.message", map[string]interface{}{"body": "context after"}) + + roomsFilter := []string{room.ID} + roomsFilterUnknown := []string{"!unknown"} + + emptyFromString := "" + fromStringValid := "1" + fromStringInvalid := "iCantBeParsed" + + testCases := []struct { + name string + wantOK bool + searchReq SearchRequest + device *userapi.Device + wantResponseCount int + from *string + }{ + { + name: "no user ID", + searchReq: SearchRequest{}, + device: &userapi.Device{}, + }, + { + name: "with alice ID", + wantOK: true, + searchReq: SearchRequest{}, + device: &aliceDevice, + }, + { + name: "searchTerm specified, found at the beginning", + wantOK: true, + searchReq: SearchRequest{ + SearchCategories: SearchCategories{RoomEvents: RoomEvents{SearchTerm: "hello"}}, + }, + device: &aliceDevice, + wantResponseCount: 1, + }, + { + name: "searchTerm specified, found at the end", + wantOK: true, + searchReq: SearchRequest{ + SearchCategories: SearchCategories{RoomEvents: RoomEvents{SearchTerm: "world3"}}, + }, + device: &aliceDevice, + wantResponseCount: 1, + }, + /* the following would need matchQuery.SetFuzziness(1) in bleve.go + { + name: "searchTerm fuzzy search", + wantOK: true, + searchReq: SearchRequest{ + SearchCategories: SearchCategories{RoomEvents: RoomEvents{SearchTerm: "hell"}}, // this still should find hello world + }, + device: &aliceDevice, + wantResponseCount: 1, + }, + */ + { + name: "searchTerm specified but no result", + wantOK: true, + searchReq: SearchRequest{ + SearchCategories: SearchCategories{RoomEvents: RoomEvents{SearchTerm: "i don't match"}}, + }, + device: &aliceDevice, + }, + { + name: "filter on room", + wantOK: true, + searchReq: SearchRequest{ + SearchCategories: SearchCategories{ + RoomEvents: RoomEvents{ + SearchTerm: "hello", + Filter: gomatrixserverlib.RoomEventFilter{ + Rooms: &roomsFilter, + }, + }, + }, + }, + device: &aliceDevice, + wantResponseCount: 1, + }, + { + name: "filter on unknown room", + searchReq: SearchRequest{ + SearchCategories: SearchCategories{ + RoomEvents: RoomEvents{ + SearchTerm: "hello", + Filter: gomatrixserverlib.RoomEventFilter{ + Rooms: &roomsFilterUnknown, + }, + }, + }, + }, + device: &aliceDevice, + }, + { + name: "include state", + wantOK: true, + searchReq: SearchRequest{ + SearchCategories: SearchCategories{ + RoomEvents: RoomEvents{ + SearchTerm: "hello", + Filter: gomatrixserverlib.RoomEventFilter{ + Rooms: &roomsFilter, + }, + IncludeState: true, + }, + }, + }, + device: &aliceDevice, + wantResponseCount: 1, + }, + { + name: "empty from does not error", + wantOK: true, + searchReq: SearchRequest{ + SearchCategories: SearchCategories{ + RoomEvents: RoomEvents{ + SearchTerm: "hello", + Filter: gomatrixserverlib.RoomEventFilter{ + Rooms: &roomsFilter, + }, + }, + }, + }, + wantResponseCount: 1, + device: &aliceDevice, + from: &emptyFromString, + }, + { + name: "valid from does not error", + wantOK: true, + searchReq: SearchRequest{ + SearchCategories: SearchCategories{ + RoomEvents: RoomEvents{ + SearchTerm: "hello", + Filter: gomatrixserverlib.RoomEventFilter{ + Rooms: &roomsFilter, + }, + }, + }, + }, + wantResponseCount: 1, + device: &aliceDevice, + from: &fromStringValid, + }, + { + name: "invalid from does error", + searchReq: SearchRequest{ + SearchCategories: SearchCategories{ + RoomEvents: RoomEvents{ + SearchTerm: "hello", + Filter: gomatrixserverlib.RoomEventFilter{ + Rooms: &roomsFilter, + }, + }, + }, + }, + device: &aliceDevice, + from: &fromStringInvalid, + }, + { + name: "order by stream position", + wantOK: true, + searchReq: SearchRequest{ + SearchCategories: SearchCategories{RoomEvents: RoomEvents{SearchTerm: "hello", OrderBy: "recent"}}, + }, + device: &aliceDevice, + wantResponseCount: 1, + }, + } + + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + cfg, processCtx, closeDB := testrig.CreateConfig(t, dbType) + defer closeDB() + + // create requisites + fts, err := fulltext.New(processCtx, cfg.SyncAPI.Fulltext) + assert.NoError(t, err) + assert.NotNil(t, fts) + + cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) + db, err := storage.NewSyncServerDatasource(processCtx.Context(), cm, &cfg.SyncAPI.Database) + assert.NoError(t, err) + + elements := []fulltext.IndexElement{} + // store the events in the database + var sp types.StreamPosition + for _, x := range room.Events() { + var stateEvents []*gomatrixserverlib.HeaderedEvent + var stateEventIDs []string + if x.Type() == gomatrixserverlib.MRoomMember { + stateEvents = append(stateEvents, x) + stateEventIDs = append(stateEventIDs, x.EventID()) + } + sp, err = db.WriteEvent(processCtx.Context(), x, stateEvents, stateEventIDs, nil, nil, false, gomatrixserverlib.HistoryVisibilityShared) + assert.NoError(t, err) + if x.Type() != "m.room.message" { + continue + } + elements = append(elements, fulltext.IndexElement{ + EventID: x.EventID(), + RoomID: x.RoomID(), + Content: string(x.Content()), + ContentType: x.Type(), + StreamPosition: int64(sp), + }) + } + // Index the events + err = fts.Index(elements...) + assert.NoError(t, err) + + // run the tests + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + reqBody := &bytes.Buffer{} + err = json.NewEncoder(reqBody).Encode(tc.searchReq) + assert.NoError(t, err) + req := httptest.NewRequest(http.MethodPost, "/", reqBody) + + res := Search(req, tc.device, db, fts, tc.from) + if !tc.wantOK && !res.Is2xx() { + return + } + resp, ok := res.JSON.(SearchResponse) + if !ok && !tc.wantOK { + t.Fatalf("not a SearchResponse: %T: %s", res.JSON, res.JSON) + } + assert.Equal(t, tc.wantResponseCount, resp.SearchCategories.RoomEvents.Count) + + // if we requested state, it should not be empty + if tc.searchReq.SearchCategories.RoomEvents.IncludeState { + assert.NotEmpty(t, resp.SearchCategories.RoomEvents.State) + } + }) + } + }) +} From fa7710315a00f6e857bb9c315c0a7ba248288b79 Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Mon, 27 Mar 2023 15:39:33 +0200 Subject: [PATCH 17/42] Add tests for the Dendrite admin APIs (#3028) Contains a breaking change, since the endpoints `/_dendrite/admin/evacuateRoom/{roomID}` and `/_dendrite/admin/evacuateUser/{userID}` are now using `POST` instead of `GET` --- clientapi/admin_test.go | 326 +++++++++++++++---- clientapi/routing/admin.go | 31 +- clientapi/routing/routing.go | 8 +- docs/administration/4_adminapi.md | 4 +- roomserver/internal/perform/perform_admin.go | 3 +- 5 files changed, 284 insertions(+), 88 deletions(-) diff --git a/clientapi/admin_test.go b/clientapi/admin_test.go index 4d2bf67b2d..3e7cb875ca 100644 --- a/clientapi/admin_test.go +++ b/clientapi/admin_test.go @@ -4,6 +4,7 @@ import ( "context" "net/http" "net/http/httptest" + "reflect" "testing" "time" @@ -14,6 +15,7 @@ import ( "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver" "github.com/matrix-org/dendrite/roomserver/api" + basepkg "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/jetstream" "github.com/matrix-org/dendrite/syncapi" @@ -57,34 +59,7 @@ func TestAdminResetPassword(t *testing.T) { bob: "", vhUser: "", } - for u := range accessTokens { - localpart, serverName, _ := gomatrixserverlib.SplitID('@', u.ID) - userRes := &uapi.PerformAccountCreationResponse{} - password := util.RandomString(8) - if err := userAPI.PerformAccountCreation(ctx, &uapi.PerformAccountCreationRequest{ - AccountType: u.AccountType, - Localpart: localpart, - ServerName: serverName, - Password: password, - }, userRes); err != nil { - t.Errorf("failed to create account: %s", err) - } - - req := test.NewRequest(t, http.MethodPost, "/_matrix/client/v3/login", test.WithJSONBody(t, map[string]interface{}{ - "type": authtypes.LoginTypePassword, - "identifier": map[string]interface{}{ - "type": "m.id.user", - "user": u.ID, - }, - "password": password, - })) - rec := httptest.NewRecorder() - routers.Client.ServeHTTP(rec, req) - if rec.Code != http.StatusOK { - t.Fatalf("failed to login: %s", rec.Body.String()) - } - accessTokens[u] = gjson.GetBytes(rec.Body.Bytes(), "access_token").String() - } + createAccessTokens(t, accessTokens, userAPI, ctx, routers) testCases := []struct { name string @@ -182,34 +157,7 @@ func TestPurgeRoom(t *testing.T) { accessTokens := map[*test.User]string{ aliceAdmin: "", } - for u := range accessTokens { - localpart, serverName, _ := gomatrixserverlib.SplitID('@', u.ID) - userRes := &uapi.PerformAccountCreationResponse{} - password := util.RandomString(8) - if err := userAPI.PerformAccountCreation(ctx, &uapi.PerformAccountCreationRequest{ - AccountType: u.AccountType, - Localpart: localpart, - ServerName: serverName, - Password: password, - }, userRes); err != nil { - t.Errorf("failed to create account: %s", err) - } - - req := test.NewRequest(t, http.MethodPost, "/_matrix/client/v3/login", test.WithJSONBody(t, map[string]interface{}{ - "type": authtypes.LoginTypePassword, - "identifier": map[string]interface{}{ - "type": "m.id.user", - "user": u.ID, - }, - "password": password, - })) - rec := httptest.NewRecorder() - routers.Client.ServeHTTP(rec, req) - if rec.Code != http.StatusOK { - t.Fatalf("failed to login: %s", rec.Body.String()) - } - accessTokens[u] = gjson.GetBytes(rec.Body.Bytes(), "access_token").String() - } + createAccessTokens(t, accessTokens, userAPI, ctx, routers) testCases := []struct { name string @@ -239,3 +187,269 @@ func TestPurgeRoom(t *testing.T) { }) } + +func TestAdminEvacuateRoom(t *testing.T) { + aliceAdmin := test.NewUser(t, test.WithAccountType(uapi.AccountTypeAdmin)) + bob := test.NewUser(t) + room := test.NewRoom(t, aliceAdmin) + + // Join Bob + room.CreateAndInsert(t, bob, gomatrixserverlib.MRoomMember, map[string]interface{}{ + "membership": "join", + }, test.WithStateKey(bob.ID)) + + ctx := context.Background() + + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + cfg, processCtx, close := testrig.CreateConfig(t, dbType) + caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) + natsInstance := jetstream.NATSInstance{} + defer close() + + routers := httputil.NewRouters() + cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) + rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + + // this starts the JetStream consumers + fsAPI := federationapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, rsAPI, caches, nil, true) + rsAPI.SetFederationAPI(fsAPI, nil) + + // Create the room + if err := api.SendEvents(ctx, rsAPI, api.KindNew, room.Events(), "test", "test", api.DoNotSendToOtherServers, nil, false); err != nil { + t.Fatalf("failed to send events: %v", err) + } + + // We mostly need the rsAPI for this test, so nil for other APIs/caches etc. + AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) + + // Create the users in the userapi and login + accessTokens := map[*test.User]string{ + aliceAdmin: "", + } + createAccessTokens(t, accessTokens, userAPI, ctx, routers) + + testCases := []struct { + name string + roomID string + wantOK bool + wantAffected []string + }{ + {name: "Can evacuate existing room", wantOK: true, roomID: room.ID, wantAffected: []string{aliceAdmin.ID, bob.ID}}, + {name: "Can not evacuate non-existent room", wantOK: false, roomID: "!doesnotexist:localhost", wantAffected: []string{}}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + req := test.NewRequest(t, http.MethodPost, "/_dendrite/admin/evacuateRoom/"+tc.roomID) + + req.Header.Set("Authorization", "Bearer "+accessTokens[aliceAdmin]) + + rec := httptest.NewRecorder() + routers.DendriteAdmin.ServeHTTP(rec, req) + t.Logf("%s", rec.Body.String()) + if tc.wantOK && rec.Code != http.StatusOK { + t.Fatalf("expected http status %d, got %d: %s", http.StatusOK, rec.Code, rec.Body.String()) + } + + affectedArr := gjson.GetBytes(rec.Body.Bytes(), "affected").Array() + affected := make([]string, 0, len(affectedArr)) + for _, x := range affectedArr { + affected = append(affected, x.Str) + } + if !reflect.DeepEqual(affected, tc.wantAffected) { + t.Fatalf("expected affected %#v, but got %#v", tc.wantAffected, affected) + } + }) + } + }) +} + +func TestAdminEvacuateUser(t *testing.T) { + aliceAdmin := test.NewUser(t, test.WithAccountType(uapi.AccountTypeAdmin)) + bob := test.NewUser(t) + room := test.NewRoom(t, aliceAdmin) + room2 := test.NewRoom(t, aliceAdmin) + + // Join Bob + room.CreateAndInsert(t, bob, gomatrixserverlib.MRoomMember, map[string]interface{}{ + "membership": "join", + }, test.WithStateKey(bob.ID)) + room2.CreateAndInsert(t, bob, gomatrixserverlib.MRoomMember, map[string]interface{}{ + "membership": "join", + }, test.WithStateKey(bob.ID)) + + ctx := context.Background() + + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + cfg, processCtx, close := testrig.CreateConfig(t, dbType) + caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) + natsInstance := jetstream.NATSInstance{} + defer close() + + routers := httputil.NewRouters() + cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) + rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + + // this starts the JetStream consumers + fsAPI := federationapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, basepkg.CreateFederationClient(cfg, nil), rsAPI, caches, nil, true) + rsAPI.SetFederationAPI(fsAPI, nil) + + // Create the room + if err := api.SendEvents(ctx, rsAPI, api.KindNew, room.Events(), "test", "test", api.DoNotSendToOtherServers, nil, false); err != nil { + t.Fatalf("failed to send events: %v", err) + } + if err := api.SendEvents(ctx, rsAPI, api.KindNew, room2.Events(), "test", "test", api.DoNotSendToOtherServers, nil, false); err != nil { + t.Fatalf("failed to send events: %v", err) + } + + // We mostly need the rsAPI for this test, so nil for other APIs/caches etc. + AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) + + // Create the users in the userapi and login + accessTokens := map[*test.User]string{ + aliceAdmin: "", + } + createAccessTokens(t, accessTokens, userAPI, ctx, routers) + + testCases := []struct { + name string + userID string + wantOK bool + wantAffectedRooms []string + }{ + {name: "Can evacuate existing user", wantOK: true, userID: bob.ID, wantAffectedRooms: []string{room.ID, room2.ID}}, + {name: "invalid userID is rejected", wantOK: false, userID: "!notauserid:test", wantAffectedRooms: []string{}}, + {name: "Can not evacuate user from different server", wantOK: false, userID: "@doesnotexist:localhost", wantAffectedRooms: []string{}}, + {name: "Can not evacuate non-existent user", wantOK: false, userID: "@doesnotexist:test", wantAffectedRooms: []string{}}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + req := test.NewRequest(t, http.MethodPost, "/_dendrite/admin/evacuateUser/"+tc.userID) + + req.Header.Set("Authorization", "Bearer "+accessTokens[aliceAdmin]) + + rec := httptest.NewRecorder() + routers.DendriteAdmin.ServeHTTP(rec, req) + t.Logf("%s", rec.Body.String()) + if tc.wantOK && rec.Code != http.StatusOK { + t.Fatalf("expected http status %d, got %d: %s", http.StatusOK, rec.Code, rec.Body.String()) + } + + affectedArr := gjson.GetBytes(rec.Body.Bytes(), "affected").Array() + affected := make([]string, 0, len(affectedArr)) + for _, x := range affectedArr { + affected = append(affected, x.Str) + } + if !reflect.DeepEqual(affected, tc.wantAffectedRooms) { + t.Fatalf("expected affected %#v, but got %#v", tc.wantAffectedRooms, affected) + } + + }) + } + // Wait for the FS API to have consumed every message + js, _ := natsInstance.Prepare(processCtx, &cfg.Global.JetStream) + timeout := time.After(time.Second) + for { + select { + case <-timeout: + t.Fatalf("FS API didn't process all events in time") + default: + } + info, err := js.ConsumerInfo(cfg.Global.JetStream.Prefixed(jetstream.OutputRoomEvent), cfg.Global.JetStream.Durable("FederationAPIRoomServerConsumer")+"Pull") + if err != nil { + time.Sleep(time.Millisecond * 10) + continue + } + if info.NumPending == 0 && info.NumAckPending == 0 { + break + } + } + }) +} + +func TestAdminMarkAsStale(t *testing.T) { + aliceAdmin := test.NewUser(t, test.WithAccountType(uapi.AccountTypeAdmin)) + + ctx := context.Background() + + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + cfg, processCtx, close := testrig.CreateConfig(t, dbType) + caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) + natsInstance := jetstream.NATSInstance{} + defer close() + + routers := httputil.NewRouters() + cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) + rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + + // We mostly need the rsAPI for this test, so nil for other APIs/caches etc. + AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) + + // Create the users in the userapi and login + accessTokens := map[*test.User]string{ + aliceAdmin: "", + } + createAccessTokens(t, accessTokens, userAPI, ctx, routers) + + testCases := []struct { + name string + userID string + wantOK bool + }{ + {name: "local user is not allowed", userID: aliceAdmin.ID}, + {name: "invalid userID", userID: "!notvalid:test"}, + {name: "remote user is allowed", userID: "@alice:localhost", wantOK: true}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + req := test.NewRequest(t, http.MethodPost, "/_dendrite/admin/refreshDevices/"+tc.userID) + + req.Header.Set("Authorization", "Bearer "+accessTokens[aliceAdmin]) + + rec := httptest.NewRecorder() + routers.DendriteAdmin.ServeHTTP(rec, req) + t.Logf("%s", rec.Body.String()) + if tc.wantOK && rec.Code != http.StatusOK { + t.Fatalf("expected http status %d, got %d: %s", http.StatusOK, rec.Code, rec.Body.String()) + } + }) + } + }) +} + +func createAccessTokens(t *testing.T, accessTokens map[*test.User]string, userAPI uapi.UserInternalAPI, ctx context.Context, routers httputil.Routers) { + t.Helper() + for u := range accessTokens { + localpart, serverName, _ := gomatrixserverlib.SplitID('@', u.ID) + userRes := &uapi.PerformAccountCreationResponse{} + password := util.RandomString(8) + if err := userAPI.PerformAccountCreation(ctx, &uapi.PerformAccountCreationRequest{ + AccountType: u.AccountType, + Localpart: localpart, + ServerName: serverName, + Password: password, + }, userRes); err != nil { + t.Errorf("failed to create account: %s", err) + } + + req := test.NewRequest(t, http.MethodPost, "/_matrix/client/v3/login", test.WithJSONBody(t, map[string]interface{}{ + "type": authtypes.LoginTypePassword, + "identifier": map[string]interface{}{ + "type": "m.id.user", + "user": u.ID, + }, + "password": password, + })) + rec := httptest.NewRecorder() + routers.Client.ServeHTTP(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("failed to login: %s", rec.Body.String()) + } + accessTokens[u] = gjson.GetBytes(rec.Body.Bytes(), "access_token").String() + } +} diff --git a/clientapi/routing/admin.go b/clientapi/routing/admin.go index a01f6b9443..76e18f2f8d 100644 --- a/clientapi/routing/admin.go +++ b/clientapi/routing/admin.go @@ -22,23 +22,16 @@ import ( "github.com/matrix-org/dendrite/userapi/api" ) -func AdminEvacuateRoom(req *http.Request, cfg *config.ClientAPI, device *api.Device, rsAPI roomserverAPI.ClientRoomserverAPI) util.JSONResponse { +func AdminEvacuateRoom(req *http.Request, rsAPI roomserverAPI.ClientRoomserverAPI) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } - roomID, ok := vars["roomID"] - if !ok { - return util.JSONResponse{ - Code: http.StatusBadRequest, - JSON: jsonerror.MissingArgument("Expecting room ID."), - } - } res := &roomserverAPI.PerformAdminEvacuateRoomResponse{} if err := rsAPI.PerformAdminEvacuateRoom( req.Context(), &roomserverAPI.PerformAdminEvacuateRoomRequest{ - RoomID: roomID, + RoomID: vars["roomID"], }, res, ); err != nil { @@ -55,18 +48,13 @@ func AdminEvacuateRoom(req *http.Request, cfg *config.ClientAPI, device *api.Dev } } -func AdminEvacuateUser(req *http.Request, cfg *config.ClientAPI, device *api.Device, rsAPI roomserverAPI.ClientRoomserverAPI) util.JSONResponse { +func AdminEvacuateUser(req *http.Request, cfg *config.ClientAPI, rsAPI roomserverAPI.ClientRoomserverAPI) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } - userID, ok := vars["userID"] - if !ok { - return util.JSONResponse{ - Code: http.StatusBadRequest, - JSON: jsonerror.MissingArgument("Expecting user ID."), - } - } + userID := vars["userID"] + _, domain, err := gomatrixserverlib.SplitID('@', userID) if err != nil { return util.MessageResponse(http.StatusBadRequest, err.Error()) @@ -103,13 +91,8 @@ func AdminPurgeRoom(req *http.Request, cfg *config.ClientAPI, device *api.Device if err != nil { return util.ErrorResponse(err) } - roomID, ok := vars["roomID"] - if !ok { - return util.JSONResponse{ - Code: http.StatusBadRequest, - JSON: jsonerror.MissingArgument("Expecting room ID."), - } - } + roomID := vars["roomID"] + res := &roomserverAPI.PerformAdminPurgeRoomResponse{} if err := rsAPI.PerformAdminPurgeRoom( context.Background(), diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index 6a86980da8..e261cb3aa4 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -155,15 +155,15 @@ func Setup( dendriteAdminRouter.Handle("/admin/evacuateRoom/{roomID}", httputil.MakeAdminAPI("admin_evacuate_room", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - return AdminEvacuateRoom(req, cfg, device, rsAPI) + return AdminEvacuateRoom(req, rsAPI) }), - ).Methods(http.MethodGet, http.MethodOptions) + ).Methods(http.MethodPost, http.MethodOptions) dendriteAdminRouter.Handle("/admin/evacuateUser/{userID}", httputil.MakeAdminAPI("admin_evacuate_user", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - return AdminEvacuateUser(req, cfg, device, rsAPI) + return AdminEvacuateUser(req, cfg, rsAPI) }), - ).Methods(http.MethodGet, http.MethodOptions) + ).Methods(http.MethodPost, http.MethodOptions) dendriteAdminRouter.Handle("/admin/purgeRoom/{roomID}", httputil.MakeAdminAPI("admin_purge_room", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { diff --git a/docs/administration/4_adminapi.md b/docs/administration/4_adminapi.md index 46cfac220d..b11aeb1a60 100644 --- a/docs/administration/4_adminapi.md +++ b/docs/administration/4_adminapi.md @@ -32,7 +32,7 @@ UPDATE userapi_accounts SET account_type = 3 WHERE localpart = '$localpart'; Where `$localpart` is the username only (e.g. `alice`). -## GET `/_dendrite/admin/evacuateRoom/{roomID}` +## POST `/_dendrite/admin/evacuateRoom/{roomID}` This endpoint will instruct Dendrite to part all local users from the given `roomID` in the URL. It may take some time to complete. A JSON body will be returned containing @@ -41,7 +41,7 @@ the user IDs of all affected users. If the room has an alias set (e.g. is published), the room's ID will not be visible in the URL, but it can be found as the room's "internal ID" in Element Web (Settings -> Advanced) -## GET `/_dendrite/admin/evacuateUser/{userID}` +## POST `/_dendrite/admin/evacuateUser/{userID}` This endpoint will instruct Dendrite to part the given local `userID` in the URL from all rooms which they are currently joined. A JSON body will be returned containing diff --git a/roomserver/internal/perform/perform_admin.go b/roomserver/internal/perform/perform_admin.go index 45089bdd17..0f1249114a 100644 --- a/roomserver/internal/perform/perform_admin.go +++ b/roomserver/internal/perform/perform_admin.go @@ -227,6 +227,7 @@ func (r *Admin) PerformAdminEvacuateUser( } return nil } + res.Affected = append(res.Affected, roomID) if len(outputEvents) == 0 { continue } @@ -237,8 +238,6 @@ func (r *Admin) PerformAdminEvacuateUser( } return nil } - - res.Affected = append(res.Affected, roomID) } return nil } From 69e3bd82a94470c4072637374dea82b8c2acfaec Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Mon, 27 Mar 2023 07:55:49 -0600 Subject: [PATCH 18/42] Add dendrite-demo-pinecone cypress tests --- .github/workflows/schedules.yaml | 41 +++++++++++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/.github/workflows/schedules.yaml b/.github/workflows/schedules.yaml index 2545949122..e76cc82f37 100644 --- a/.github/workflows/schedules.yaml +++ b/.github/workflows/schedules.yaml @@ -219,7 +219,7 @@ jobs: flags: complement fail_ci_if_error: true - element_web: + element-web: timeout-minutes: 120 runs-on: ubuntu-latest steps: @@ -257,3 +257,42 @@ jobs: env: PUPPETEER_SKIP_CHROMIUM_DOWNLOAD: true TMPDIR: ${{ runner.temp }} + + element-web-pinecone: + timeout-minutes: 120 + runs-on: ubuntu-latest + steps: + - uses: tecolicom/actions-use-apt-tools@v1 + with: + # Our test suite includes some screenshot tests with unusual diacritics, which are + # supposed to be covered by STIXGeneral. + tools: fonts-stix + - uses: actions/checkout@v2 + with: + repository: matrix-org/matrix-react-sdk + - uses: actions/setup-node@v3 + with: + cache: 'yarn' + - name: Fetch layered build + run: scripts/ci/layered.sh + - name: Copy config + run: cp element.io/develop/config.json config.json + working-directory: ./element-web + - name: Build + env: + CI_PACKAGE: true + NODE_OPTIONS: "--openssl-legacy-provider" + run: yarn build + working-directory: ./element-web + - name: Edit Test Config + run: | + sed -i '/HOMESERVER/c\ HOMESERVER: "dendritePinecone",' cypress.config.ts + - name: "Run cypress tests" + uses: cypress-io/github-action@v4.1.1 + with: + browser: chrome + start: npx serve -p 8080 ./element-web/webapp + wait-on: 'http://localhost:8080' + env: + PUPPETEER_SKIP_CHROMIUM_DOWNLOAD: true + TMPDIR: ${{ runner.temp }} From f4104b4b5d21df9dfedd9873a5d8b6bc63e96e99 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Mon, 27 Mar 2023 17:19:53 -0600 Subject: [PATCH 19/42] Pinecone-demo: Wait on dendrite before shutting down --- cmd/dendrite-demo-pinecone/monolith/monolith.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/dendrite-demo-pinecone/monolith/monolith.go b/cmd/dendrite-demo-pinecone/monolith/monolith.go index 10a3493e12..d1a6e39e98 100644 --- a/cmd/dendrite-demo-pinecone/monolith/monolith.go +++ b/cmd/dendrite-demo-pinecone/monolith/monolith.go @@ -213,7 +213,7 @@ func (p *P2PMonolith) Stop() { } func (p *P2PMonolith) WaitForShutdown() { - p.ProcessCtx.WaitForShutdown() + base.WaitForShutdown(p.ProcessCtx) p.closeAllResources() } From 28d3e296a8adaeea34bca01c7e4e6a0e22918390 Mon Sep 17 00:00:00 2001 From: Rhea Danzey Date: Tue, 28 Mar 2023 01:30:19 -0500 Subject: [PATCH 20/42] Rdanzey/helm-fixes-existing-db-secrets (#3033) Fixes some Helm templating issues when setting up a deployment with an existing database / signing keys. - Allows for `.Values.postgresql.enabled: false` as long as `.Values.global.dendrite_config.database.connection_string` is defined - Allows for '.Values.signing_key.create: false' if `.Values.signing_key.existingSecret` is set Also fixes an error in the template resulting in profiling port not being set correctly: ``` Error: template: dendrite-meta/charts/dendrite/templates/deployment.yaml:60:35: executing "dendrite-meta/charts/dendrite/templates/deployment.yaml" at <$.Values.global.profiling.port>: nil pointer evaluating interface {}.port ``` ### Pull Request Checklist * [x] I have added Go unit tests or [Complement integration tests](https://github.com/matrix-org/complement) for this PR _or_ I have justified why this PR doesn't need tests - Helm template fixes, no golang changes * [x] Pull request includes a [sign off below using a legally identifiable name](https://matrix-org.github.io/dendrite/development/contributing#sign-off) _or_ I have already signed off privately Signed-off-by: Rhea Danzey --------- Signed-off-by: Rhea Danzey Co-authored-by: Till Faelligen <2353100+S7evinK@users.noreply.github.com> --- helm/dendrite/Chart.yaml | 2 +- helm/dendrite/templates/_helpers.tpl | 14 ++++---------- helm/dendrite/templates/deployment.yaml | 8 ++------ 3 files changed, 7 insertions(+), 17 deletions(-) diff --git a/helm/dendrite/Chart.yaml b/helm/dendrite/Chart.yaml index b352601e84..3ef45a6df6 100644 --- a/helm/dendrite/Chart.yaml +++ b/helm/dendrite/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 name: dendrite -version: "0.12.0" +version: "0.12.1" appVersion: "0.12.0" description: Dendrite Matrix Homeserver type: application diff --git a/helm/dendrite/templates/_helpers.tpl b/helm/dendrite/templates/_helpers.tpl index 0267065888..36bcefd8f1 100644 --- a/helm/dendrite/templates/_helpers.tpl +++ b/helm/dendrite/templates/_helpers.tpl @@ -1,15 +1,9 @@ {{- define "validate.config" }} -{{- if not .Values.signing_key.create -}} -{{- fail "You must create a signing key for configuration.signing_key. (see https://github.com/matrix-org/dendrite/blob/master/docs/INSTALL.md#server-key-generation)" -}} +{{- if and (not .Values.signing_key.create) (eq .Values.signing_key.existingSecret "") -}} +{{- fail "You must create a signing key for configuration.signing_key OR specify an existing secret name in .Values.signing_key.existingSecret to mount it. (see https://github.com/matrix-org/dendrite/blob/master/docs/INSTALL.md#server-key-generation)" -}} {{- end -}} -{{- if not (or .Values.dendrite_config.global.database.host .Values.postgresql.enabled) -}} -{{- fail "Database server must be set." -}} -{{- end -}} -{{- if not (or .Values.dendrite_config.global.database.user .Values.postgresql.enabled) -}} -{{- fail "Database user must be set." -}} -{{- end -}} -{{- if not (or .Values.dendrite_config.global.database.password .Values.postgresql.enabled) -}} -{{- fail "Database password must be set." -}} +{{- if and (not .Values.postgresql.enabled) (eq .Values.dendrite_config.global.database.connection_string "") -}} +{{- fail "Database connection string must be set." -}} {{- end -}} {{- end -}} diff --git a/helm/dendrite/templates/deployment.yaml b/helm/dendrite/templates/deployment.yaml index b463c7d0b9..2a0f3a9e92 100644 --- a/helm/dendrite/templates/deployment.yaml +++ b/helm/dendrite/templates/deployment.yaml @@ -17,11 +17,7 @@ spec: labels: {{- include "dendrite.selectorLabels" . | nindent 8 }} annotations: - confighash-global: secret-{{ .Values.global | toYaml | sha256sum | trunc 32 }} - confighash-clientapi: clientapi-{{ .Values.clientapi | toYaml | sha256sum | trunc 32 }} - confighash-federationapi: federationapi-{{ .Values.federationapi | toYaml | sha256sum | trunc 32 }} - confighash-mediaapi: mediaapi-{{ .Values.mediaapi | toYaml | sha256sum | trunc 32 }} - confighash-syncapi: syncapi-{{ .Values.syncapi | toYaml | sha256sum | trunc 32 }} + confighash: secret-{{ .Values.dendrite_config | toYaml | sha256sum | trunc 32 }} spec: volumes: - name: {{ include "dendrite.fullname" . }}-conf-vol @@ -57,7 +53,7 @@ spec: {{- if $.Values.dendrite_config.global.profiling.enabled }} env: - name: PPROFLISTEN - value: "localhost:{{- $.Values.global.profiling.port -}}" + value: "localhost:{{- $.Values.dendrite_config.global.profiling.port -}}" {{- end }} resources: {{- toYaml $.Values.resources | nindent 10 }} From 2854ffeb7d933f76cbdfe0eb6775a8f5699f4170 Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Fri, 31 Mar 2023 10:15:01 +0200 Subject: [PATCH 21/42] Add CS API device tests (#3029) Adds tests for - `/devices` - `/delete_devices` (also adds UIA) --- clientapi/admin_test.go | 67 ++---- clientapi/clientapi_test.go | 373 ++++++++++++++++++++++++++++++++ clientapi/routing/deactivate.go | 5 +- clientapi/routing/device.go | 40 +++- clientapi/routing/routing.go | 2 +- userapi/api/api.go | 1 - userapi/internal/user_api.go | 5 - 7 files changed, 422 insertions(+), 71 deletions(-) create mode 100644 clientapi/clientapi_test.go diff --git a/clientapi/admin_test.go b/clientapi/admin_test.go index 3e7cb875ca..79e88d137e 100644 --- a/clientapi/admin_test.go +++ b/clientapi/admin_test.go @@ -8,7 +8,6 @@ import ( "testing" "time" - "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/federationapi" "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/internal/httputil" @@ -54,10 +53,10 @@ func TestAdminResetPassword(t *testing.T) { AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) // Create the users in the userapi and login - accessTokens := map[*test.User]string{ - aliceAdmin: "", - bob: "", - vhUser: "", + accessTokens := map[*test.User]userDevice{ + aliceAdmin: {}, + bob: {}, + vhUser: {}, } createAccessTokens(t, accessTokens, userAPI, ctx, routers) @@ -103,7 +102,7 @@ func TestAdminResetPassword(t *testing.T) { } if tc.withHeader { - req.Header.Set("Authorization", "Bearer "+accessTokens[tc.requestingUser]) + req.Header.Set("Authorization", "Bearer "+accessTokens[tc.requestingUser].accessToken) } rec := httptest.NewRecorder() @@ -154,8 +153,8 @@ func TestPurgeRoom(t *testing.T) { AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) // Create the users in the userapi and login - accessTokens := map[*test.User]string{ - aliceAdmin: "", + accessTokens := map[*test.User]userDevice{ + aliceAdmin: {}, } createAccessTokens(t, accessTokens, userAPI, ctx, routers) @@ -174,7 +173,7 @@ func TestPurgeRoom(t *testing.T) { t.Run(tc.name, func(t *testing.T) { req := test.NewRequest(t, http.MethodPost, "/_dendrite/admin/purgeRoom/"+tc.roomID) - req.Header.Set("Authorization", "Bearer "+accessTokens[aliceAdmin]) + req.Header.Set("Authorization", "Bearer "+accessTokens[aliceAdmin].accessToken) rec := httptest.NewRecorder() routers.DendriteAdmin.ServeHTTP(rec, req) @@ -224,8 +223,8 @@ func TestAdminEvacuateRoom(t *testing.T) { AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) // Create the users in the userapi and login - accessTokens := map[*test.User]string{ - aliceAdmin: "", + accessTokens := map[*test.User]userDevice{ + aliceAdmin: {}, } createAccessTokens(t, accessTokens, userAPI, ctx, routers) @@ -243,7 +242,7 @@ func TestAdminEvacuateRoom(t *testing.T) { t.Run(tc.name, func(t *testing.T) { req := test.NewRequest(t, http.MethodPost, "/_dendrite/admin/evacuateRoom/"+tc.roomID) - req.Header.Set("Authorization", "Bearer "+accessTokens[aliceAdmin]) + req.Header.Set("Authorization", "Bearer "+accessTokens[aliceAdmin].accessToken) rec := httptest.NewRecorder() routers.DendriteAdmin.ServeHTTP(rec, req) @@ -308,8 +307,8 @@ func TestAdminEvacuateUser(t *testing.T) { AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) // Create the users in the userapi and login - accessTokens := map[*test.User]string{ - aliceAdmin: "", + accessTokens := map[*test.User]userDevice{ + aliceAdmin: {}, } createAccessTokens(t, accessTokens, userAPI, ctx, routers) @@ -329,7 +328,7 @@ func TestAdminEvacuateUser(t *testing.T) { t.Run(tc.name, func(t *testing.T) { req := test.NewRequest(t, http.MethodPost, "/_dendrite/admin/evacuateUser/"+tc.userID) - req.Header.Set("Authorization", "Bearer "+accessTokens[aliceAdmin]) + req.Header.Set("Authorization", "Bearer "+accessTokens[aliceAdmin].accessToken) rec := httptest.NewRecorder() routers.DendriteAdmin.ServeHTTP(rec, req) @@ -390,8 +389,8 @@ func TestAdminMarkAsStale(t *testing.T) { AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) // Create the users in the userapi and login - accessTokens := map[*test.User]string{ - aliceAdmin: "", + accessTokens := map[*test.User]userDevice{ + aliceAdmin: {}, } createAccessTokens(t, accessTokens, userAPI, ctx, routers) @@ -409,7 +408,7 @@ func TestAdminMarkAsStale(t *testing.T) { t.Run(tc.name, func(t *testing.T) { req := test.NewRequest(t, http.MethodPost, "/_dendrite/admin/refreshDevices/"+tc.userID) - req.Header.Set("Authorization", "Bearer "+accessTokens[aliceAdmin]) + req.Header.Set("Authorization", "Bearer "+accessTokens[aliceAdmin].accessToken) rec := httptest.NewRecorder() routers.DendriteAdmin.ServeHTTP(rec, req) @@ -421,35 +420,3 @@ func TestAdminMarkAsStale(t *testing.T) { } }) } - -func createAccessTokens(t *testing.T, accessTokens map[*test.User]string, userAPI uapi.UserInternalAPI, ctx context.Context, routers httputil.Routers) { - t.Helper() - for u := range accessTokens { - localpart, serverName, _ := gomatrixserverlib.SplitID('@', u.ID) - userRes := &uapi.PerformAccountCreationResponse{} - password := util.RandomString(8) - if err := userAPI.PerformAccountCreation(ctx, &uapi.PerformAccountCreationRequest{ - AccountType: u.AccountType, - Localpart: localpart, - ServerName: serverName, - Password: password, - }, userRes); err != nil { - t.Errorf("failed to create account: %s", err) - } - - req := test.NewRequest(t, http.MethodPost, "/_matrix/client/v3/login", test.WithJSONBody(t, map[string]interface{}{ - "type": authtypes.LoginTypePassword, - "identifier": map[string]interface{}{ - "type": "m.id.user", - "user": u.ID, - }, - "password": password, - })) - rec := httptest.NewRecorder() - routers.Client.ServeHTTP(rec, req) - if rec.Code != http.StatusOK { - t.Fatalf("failed to login: %s", rec.Body.String()) - } - accessTokens[u] = gjson.GetBytes(rec.Body.Bytes(), "access_token").String() - } -} diff --git a/clientapi/clientapi_test.go b/clientapi/clientapi_test.go new file mode 100644 index 0000000000..d909155265 --- /dev/null +++ b/clientapi/clientapi_test.go @@ -0,0 +1,373 @@ +package clientapi + +import ( + "bytes" + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/matrix-org/dendrite/clientapi/auth/authtypes" + "github.com/matrix-org/dendrite/internal/caching" + "github.com/matrix-org/dendrite/internal/httputil" + "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/roomserver" + "github.com/matrix-org/dendrite/setup/jetstream" + "github.com/matrix-org/dendrite/test" + "github.com/matrix-org/dendrite/test/testrig" + "github.com/matrix-org/dendrite/userapi" + uapi "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/util" + "github.com/tidwall/gjson" +) + +type userDevice struct { + accessToken string + deviceID string + password string +} + +func TestGetPutDevices(t *testing.T) { + alice := test.NewUser(t) + bob := test.NewUser(t) + + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + testCases := []struct { + name string + requestUser *test.User + deviceUser *test.User + request *http.Request + wantStatusCode int + validateFunc func(t *testing.T, device userDevice, routers httputil.Routers) + }{ + { + name: "can get all devices", + requestUser: alice, + request: httptest.NewRequest(http.MethodGet, "/_matrix/client/v3/devices", strings.NewReader("")), + wantStatusCode: http.StatusOK, + }, + { + name: "can get specific own device", + requestUser: alice, + deviceUser: alice, + request: httptest.NewRequest(http.MethodGet, "/_matrix/client/v3/devices/", strings.NewReader("")), + wantStatusCode: http.StatusOK, + }, + { + name: "can not get device for different user", + requestUser: alice, + deviceUser: bob, + request: httptest.NewRequest(http.MethodGet, "/_matrix/client/v3/devices/", strings.NewReader("")), + wantStatusCode: http.StatusNotFound, + }, + { + name: "can update own device", + requestUser: alice, + deviceUser: alice, + request: httptest.NewRequest(http.MethodPut, "/_matrix/client/v3/devices/", strings.NewReader(`{"display_name":"my new displayname"}`)), + wantStatusCode: http.StatusOK, + validateFunc: func(t *testing.T, device userDevice, routers httputil.Routers) { + req := httptest.NewRequest(http.MethodGet, "/_matrix/client/v3/devices/"+device.deviceID, strings.NewReader("")) + req.Header.Set("Authorization", "Bearer "+device.accessToken) + rec := httptest.NewRecorder() + routers.Client.ServeHTTP(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("expected HTTP 200, got %d: %s", rec.Code, rec.Body.String()) + } + gotDisplayName := gjson.GetBytes(rec.Body.Bytes(), "display_name").Str + if gotDisplayName != "my new displayname" { + t.Fatalf("expected displayname '%s', got '%s'", "my new displayname", gotDisplayName) + } + }, + }, + { + // this should return "device does not exist" + name: "can not update device for different user", + requestUser: alice, + deviceUser: bob, + request: httptest.NewRequest(http.MethodPut, "/_matrix/client/v3/devices/", strings.NewReader(`{"display_name":"my new displayname"}`)), + wantStatusCode: http.StatusNotFound, + }, + } + + cfg, processCtx, close := testrig.CreateConfig(t, dbType) + caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) + natsInstance := jetstream.NATSInstance{} + defer close() + + routers := httputil.NewRouters() + cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) + rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + + // We mostly need the rsAPI for this test, so nil for other APIs/caches etc. + AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) + + accessTokens := map[*test.User]userDevice{ + alice: {}, + bob: {}, + } + createAccessTokens(t, accessTokens, userAPI, processCtx.Context(), routers) + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + dev := accessTokens[tc.requestUser] + if tc.deviceUser != nil { + tc.request = httptest.NewRequest(tc.request.Method, tc.request.RequestURI+accessTokens[tc.deviceUser].deviceID, tc.request.Body) + } + tc.request.Header.Set("Authorization", "Bearer "+dev.accessToken) + rec := httptest.NewRecorder() + routers.Client.ServeHTTP(rec, tc.request) + if rec.Code != tc.wantStatusCode { + t.Fatalf("expected HTTP 200, got %d: %s", rec.Code, rec.Body.String()) + } + if tc.wantStatusCode != http.StatusOK && rec.Code != http.StatusOK { + return + } + if tc.validateFunc != nil { + tc.validateFunc(t, dev, routers) + } + }) + } + }) +} + +// Deleting devices requires the UIA dance, so do this in a different test +func TestDeleteDevice(t *testing.T) { + alice := test.NewUser(t) + localpart, serverName, _ := gomatrixserverlib.SplitID('@', alice.ID) + + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + cfg, processCtx, closeDB := testrig.CreateConfig(t, dbType) + defer closeDB() + + natsInstance := jetstream.NATSInstance{} + routers := httputil.NewRouters() + cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) + caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) + rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + + // We mostly need the rsAPI/ for this test, so nil for other APIs/caches etc. + AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) + + accessTokens := map[*test.User]userDevice{ + alice: {}, + } + + // create the account and an initial device + createAccessTokens(t, accessTokens, userAPI, processCtx.Context(), routers) + + // create some more devices + accessToken := util.RandomString(8) + devRes := &uapi.PerformDeviceCreationResponse{} + if err := userAPI.PerformDeviceCreation(processCtx.Context(), &uapi.PerformDeviceCreationRequest{ + Localpart: localpart, + ServerName: serverName, + AccessToken: accessToken, + NoDeviceListUpdate: true, + }, devRes); err != nil { + t.Fatal(err) + } + if !devRes.DeviceCreated { + t.Fatalf("failed to create device") + } + secondDeviceID := devRes.Device.ID + + // initiate UIA for the second device + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodDelete, "/_matrix/client/v3/devices/"+secondDeviceID, strings.NewReader("")) + req.Header.Set("Authorization", "Bearer "+accessTokens[alice].accessToken) + routers.Client.ServeHTTP(rec, req) + if rec.Code != http.StatusUnauthorized { + t.Fatalf("expected HTTP 401, got %d: %s", rec.Code, rec.Body.String()) + } + // get the session ID + sessionID := gjson.GetBytes(rec.Body.Bytes(), "session").Str + + // prepare UIA request body + reqBody := bytes.Buffer{} + if err := json.NewEncoder(&reqBody).Encode(map[string]interface{}{ + "auth": map[string]string{ + "session": sessionID, + "type": authtypes.LoginTypePassword, + "user": alice.ID, + "password": accessTokens[alice].password, + }, + }); err != nil { + t.Fatal(err) + } + + // copy the request body, so we can use it again for the successful delete + reqBody2 := reqBody + + // do the same request again, this time with our UIA, but for a different device ID, this should fail + rec = httptest.NewRecorder() + + req = httptest.NewRequest(http.MethodDelete, "/_matrix/client/v3/devices/"+accessTokens[alice].deviceID, &reqBody) + req.Header.Set("Authorization", "Bearer "+accessTokens[alice].accessToken) + routers.Client.ServeHTTP(rec, req) + if rec.Code != http.StatusForbidden { + t.Fatalf("expected HTTP 403, got %d: %s", rec.Code, rec.Body.String()) + } + + // do the same request again, this time with our UIA, but for the correct device ID, this should be fine + rec = httptest.NewRecorder() + req = httptest.NewRequest(http.MethodDelete, "/_matrix/client/v3/devices/"+secondDeviceID, &reqBody2) + req.Header.Set("Authorization", "Bearer "+accessTokens[alice].accessToken) + routers.Client.ServeHTTP(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("expected HTTP 200, got %d: %s", rec.Code, rec.Body.String()) + } + + // verify devices are deleted + rec = httptest.NewRecorder() + req = httptest.NewRequest(http.MethodGet, "/_matrix/client/v3/devices", strings.NewReader("")) + req.Header.Set("Authorization", "Bearer "+accessTokens[alice].accessToken) + routers.Client.ServeHTTP(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("expected HTTP 200, got %d: %s", rec.Code, rec.Body.String()) + } + for _, device := range gjson.GetBytes(rec.Body.Bytes(), "devices.#.device_id").Array() { + if device.Str == secondDeviceID { + t.Fatalf("expected device %s to be deleted, but wasn't", secondDeviceID) + } + } + }) +} + +// Deleting devices requires the UIA dance, so do this in a different test +func TestDeleteDevices(t *testing.T) { + alice := test.NewUser(t) + localpart, serverName, _ := gomatrixserverlib.SplitID('@', alice.ID) + + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + cfg, processCtx, closeDB := testrig.CreateConfig(t, dbType) + defer closeDB() + + natsInstance := jetstream.NATSInstance{} + routers := httputil.NewRouters() + cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) + caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) + rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + + // We mostly need the rsAPI/ for this test, so nil for other APIs/caches etc. + AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) + + accessTokens := map[*test.User]userDevice{ + alice: {}, + } + + // create the account and an initial device + createAccessTokens(t, accessTokens, userAPI, processCtx.Context(), routers) + + // create some more devices + var devices []string + for i := 0; i < 10; i++ { + accessToken := util.RandomString(8) + devRes := &uapi.PerformDeviceCreationResponse{} + if err := userAPI.PerformDeviceCreation(processCtx.Context(), &uapi.PerformDeviceCreationRequest{ + Localpart: localpart, + ServerName: serverName, + AccessToken: accessToken, + NoDeviceListUpdate: true, + }, devRes); err != nil { + t.Fatal(err) + } + if !devRes.DeviceCreated { + t.Fatalf("failed to create device") + } + devices = append(devices, devRes.Device.ID) + } + + // initiate UIA + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPost, "/_matrix/client/v3/delete_devices", strings.NewReader("")) + req.Header.Set("Authorization", "Bearer "+accessTokens[alice].accessToken) + routers.Client.ServeHTTP(rec, req) + if rec.Code != http.StatusUnauthorized { + t.Fatalf("expected HTTP 401, got %d: %s", rec.Code, rec.Body.String()) + } + // get the session ID + sessionID := gjson.GetBytes(rec.Body.Bytes(), "session").Str + + // prepare UIA request body + reqBody := bytes.Buffer{} + if err := json.NewEncoder(&reqBody).Encode(map[string]interface{}{ + "auth": map[string]string{ + "session": sessionID, + "type": authtypes.LoginTypePassword, + "user": alice.ID, + "password": accessTokens[alice].password, + }, + "devices": devices[5:], + }); err != nil { + t.Fatal(err) + } + + // do the same request again, this time with our UIA, + rec = httptest.NewRecorder() + req = httptest.NewRequest(http.MethodPost, "/_matrix/client/v3/delete_devices", &reqBody) + req.Header.Set("Authorization", "Bearer "+accessTokens[alice].accessToken) + routers.Client.ServeHTTP(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("expected HTTP 200, got %d: %s", rec.Code, rec.Body.String()) + } + + // verify devices are deleted + rec = httptest.NewRecorder() + req = httptest.NewRequest(http.MethodGet, "/_matrix/client/v3/devices", strings.NewReader("")) + req.Header.Set("Authorization", "Bearer "+accessTokens[alice].accessToken) + routers.Client.ServeHTTP(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("expected HTTP 200, got %d: %s", rec.Code, rec.Body.String()) + } + for _, device := range gjson.GetBytes(rec.Body.Bytes(), "devices.#.device_id").Array() { + for _, deletedDevice := range devices[5:] { + if device.Str == deletedDevice { + t.Fatalf("expected device %s to be deleted, but wasn't", deletedDevice) + } + } + } + }) +} + +func createAccessTokens(t *testing.T, accessTokens map[*test.User]userDevice, userAPI uapi.UserInternalAPI, ctx context.Context, routers httputil.Routers) { + t.Helper() + for u := range accessTokens { + localpart, serverName, _ := gomatrixserverlib.SplitID('@', u.ID) + userRes := &uapi.PerformAccountCreationResponse{} + password := util.RandomString(8) + if err := userAPI.PerformAccountCreation(ctx, &uapi.PerformAccountCreationRequest{ + AccountType: u.AccountType, + Localpart: localpart, + ServerName: serverName, + Password: password, + }, userRes); err != nil { + t.Errorf("failed to create account: %s", err) + } + req := test.NewRequest(t, http.MethodPost, "/_matrix/client/v3/login", test.WithJSONBody(t, map[string]interface{}{ + "type": authtypes.LoginTypePassword, + "identifier": map[string]interface{}{ + "type": "m.id.user", + "user": u.ID, + }, + "password": password, + })) + rec := httptest.NewRecorder() + routers.Client.ServeHTTP(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("failed to login: %s", rec.Body.String()) + } + accessTokens[u] = userDevice{ + accessToken: gjson.GetBytes(rec.Body.Bytes(), "access_token").String(), + deviceID: gjson.GetBytes(rec.Body.Bytes(), "device_id").String(), + password: password, + } + } +} diff --git a/clientapi/routing/deactivate.go b/clientapi/routing/deactivate.go index f213db7f3e..3f4f539f63 100644 --- a/clientapi/routing/deactivate.go +++ b/clientapi/routing/deactivate.go @@ -33,7 +33,7 @@ func Deactivate( return *errRes } - localpart, _, err := gomatrixserverlib.SplitID('@', login.Username()) + localpart, serverName, err := gomatrixserverlib.SplitID('@', login.Username()) if err != nil { util.GetLogger(req.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed") return jsonerror.InternalServerError() @@ -41,7 +41,8 @@ func Deactivate( var res api.PerformAccountDeactivationResponse err = accountAPI.PerformAccountDeactivation(ctx, &api.PerformAccountDeactivationRequest{ - Localpart: localpart, + Localpart: localpart, + ServerName: serverName, }, &res) if err != nil { util.GetLogger(ctx).WithError(err).Error("userAPI.PerformAccountDeactivation failed") diff --git a/clientapi/routing/device.go b/clientapi/routing/device.go index e3a02661c4..331bacc3c0 100644 --- a/clientapi/routing/device.go +++ b/clientapi/routing/device.go @@ -15,6 +15,7 @@ package routing import ( + "encoding/json" "io" "net" "net/http" @@ -146,12 +147,6 @@ func UpdateDeviceByID( JSON: jsonerror.Forbidden("device does not exist"), } } - if performRes.Forbidden { - return util.JSONResponse{ - Code: http.StatusForbidden, - JSON: jsonerror.Forbidden("device not owned by current user"), - } - } return util.JSONResponse{ Code: http.StatusOK, @@ -189,7 +184,7 @@ func DeleteDeviceById( if dev != deviceID { return util.JSONResponse{ Code: http.StatusForbidden, - JSON: jsonerror.Forbidden("session & device mismatch"), + JSON: jsonerror.Forbidden("session and device mismatch"), } } } @@ -242,16 +237,37 @@ func DeleteDeviceById( // DeleteDevices handles POST requests to /delete_devices func DeleteDevices( - req *http.Request, userAPI api.ClientUserAPI, device *api.Device, + req *http.Request, userInteractiveAuth *auth.UserInteractive, userAPI api.ClientUserAPI, device *api.Device, ) util.JSONResponse { ctx := req.Context() - payload := devicesDeleteJSON{} - if resErr := httputil.UnmarshalJSONRequest(req, &payload); resErr != nil { - return *resErr + bodyBytes, err := io.ReadAll(req.Body) + if err != nil { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.BadJSON("The request body could not be read: " + err.Error()), + } } + defer req.Body.Close() // nolint:errcheck - defer req.Body.Close() // nolint: errcheck + // initiate UIA + login, errRes := userInteractiveAuth.Verify(ctx, bodyBytes, device) + if errRes != nil { + return *errRes + } + + if login.Username() != device.UserID { + return util.JSONResponse{ + Code: http.StatusForbidden, + JSON: jsonerror.Forbidden("unable to delete devices for other user"), + } + } + + payload := devicesDeleteJSON{} + if err = json.Unmarshal(bodyBytes, &payload); err != nil { + util.GetLogger(ctx).WithError(err).Error("unable to unmarshal device deletion request") + return jsonerror.InternalServerError() + } var res api.PerformDeviceDeletionResponse if err := userAPI.PerformDeviceDeletion(ctx, &api.PerformDeviceDeletionRequest{ diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index e261cb3aa4..6c8035d400 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -1115,7 +1115,7 @@ func Setup( v3mux.Handle("/delete_devices", httputil.MakeAuthAPI("delete_devices", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - return DeleteDevices(req, userAPI, device) + return DeleteDevices(req, userInteractiveAuth, userAPI, device) }), ).Methods(http.MethodPost, http.MethodOptions) diff --git a/userapi/api/api.go b/userapi/api/api.go index fa297f7735..19d4868483 100644 --- a/userapi/api/api.go +++ b/userapi/api/api.go @@ -224,7 +224,6 @@ type PerformDeviceUpdateRequest struct { } type PerformDeviceUpdateResponse struct { DeviceExists bool - Forbidden bool } type PerformDeviceDeletionRequest struct { diff --git a/userapi/internal/user_api.go b/userapi/internal/user_api.go index 8977697b58..4049d13b61 100644 --- a/userapi/internal/user_api.go +++ b/userapi/internal/user_api.go @@ -386,11 +386,6 @@ func (a *UserInternalAPI) PerformDeviceUpdate(ctx context.Context, req *api.Perf } res.DeviceExists = true - if dev.UserID != req.RequestingUserID { - res.Forbidden = true - return nil - } - err = a.DB.UpdateDevice(ctx, localpart, domain, req.DeviceID, req.DisplayName) if err != nil { util.GetLogger(ctx).WithError(err).Error("deviceDB.UpdateDevice failed") From 44ed0a327948c0c8812fe2a895af4c3dfde0c33d Mon Sep 17 00:00:00 2001 From: George Antoniadis Date: Mon, 3 Apr 2023 07:13:06 +0100 Subject: [PATCH 22/42] add deployment strategy option to helm chart (#3021) @S7evinK minor update to the helm chart on top of you existing fixes to allow setting the update strategy as the default `RollingUpdate` one is a bit annoying if using `ReadWriteOnce` volumes for media. Hope this makes sense. ### Pull Request Checklist * [x] ~~I have added Go unit tests or [Complement integration tests](https://github.com/matrix-org/complement) for this PR _or_ I have justified why this PR doesn't need tests~~ Haven't touched any go files. * [x] Pull request includes a [sign off below using a legally identifiable name](https://matrix-org.github.io/dendrite/development/contributing#sign-off) _or_ I have already signed off privately Signed-off-by: `George Antoniadis ` [skip ci] --- helm/dendrite/templates/deployment.yaml | 7 +++++++ helm/dendrite/values.yaml | 10 ++++++++++ 2 files changed, 17 insertions(+) diff --git a/helm/dendrite/templates/deployment.yaml b/helm/dendrite/templates/deployment.yaml index 2a0f3a9e92..55ae053f27 100644 --- a/helm/dendrite/templates/deployment.yaml +++ b/helm/dendrite/templates/deployment.yaml @@ -19,6 +19,13 @@ spec: annotations: confighash: secret-{{ .Values.dendrite_config | toYaml | sha256sum | trunc 32 }} spec: + strategy: + type: {{ $.Values.strategy.type }} + {{- if eq $.Values.strategy.type "RollingUpdate" }} + rollingUpdate: + maxSurge: {{ $.Values.strategy.rollingUpdate.maxSurge }} + maxUnavailable: {{ $.Values.strategy.rollingUpdate.maxUnavailable }} + {{- end }} volumes: - name: {{ include "dendrite.fullname" . }}-conf-vol secret: diff --git a/helm/dendrite/values.yaml b/helm/dendrite/values.yaml index c219d27f8b..d6be2cdbbb 100644 --- a/helm/dendrite/values.yaml +++ b/helm/dendrite/values.yaml @@ -43,6 +43,16 @@ persistence: # -- PVC Storage Request for the search volume capacity: "1Gi" +strategy: + # -- Strategy to use for rolling updates (e.g. Recreate, RollingUpdate) + # If you are using ReadWriteOnce volumes, you should probably use Recreate + type: RollingUpdate + rollingUpdate: + # -- Maximum number of pods that can be unavailable during the update process + maxUnavailable: 25% + # -- Maximum number of pods that can be scheduled above the desired number of pods + maxSurge: 25% + dendrite_config: version: 2 global: From 01dd02dad2c76a8bb6d28e3456bf8df40e16d113 Mon Sep 17 00:00:00 2001 From: Rhea Danzey Date: Mon, 3 Apr 2023 02:00:32 -0500 Subject: [PATCH 23/42] chart - Add configuration for extra volumes / volume mounts (#3042) Adds configuration for additional volumes / volumeMounts to the Dendrite pod to inject configuration / secrets outside of the chart's templates ### Pull Request Checklist * [x] I have added Go unit tests or [Complement integration tests](https://github.com/matrix-org/complement) for this PR _or_ I have justified why this PR doesn't need tests - Helm chart changes * [x] Pull request includes a [sign off below using a legally identifiable name](https://matrix-org.github.io/dendrite/development/contributing#sign-off) _or_ I have already signed off privately Signed-off-by: Rhea Danzey --------- Signed-off-by: Rhea Danzey Co-authored-by: Till <2353100+S7evinK@users.noreply.github.com> [skip ci] --- helm/dendrite/Chart.yaml | 2 +- helm/dendrite/templates/deployment.yaml | 20 +++++++++++++------- helm/dendrite/values.yaml | 14 ++++++++++++++ 3 files changed, 28 insertions(+), 8 deletions(-) diff --git a/helm/dendrite/Chart.yaml b/helm/dendrite/Chart.yaml index 3ef45a6df6..6a428e00f6 100644 --- a/helm/dendrite/Chart.yaml +++ b/helm/dendrite/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 name: dendrite -version: "0.12.1" +version: "0.12.2" appVersion: "0.12.0" description: Dendrite Matrix Homeserver type: application diff --git a/helm/dendrite/templates/deployment.yaml b/helm/dendrite/templates/deployment.yaml index 55ae053f27..df7dbbdc3d 100644 --- a/helm/dendrite/templates/deployment.yaml +++ b/helm/dendrite/templates/deployment.yaml @@ -12,6 +12,13 @@ spec: matchLabels: {{- include "dendrite.selectorLabels" . | nindent 6 }} replicas: 1 + strategy: + type: {{ $.Values.strategy.type }} + {{- if eq $.Values.strategy.type "RollingUpdate" }} + rollingUpdate: + maxSurge: {{ $.Values.strategy.rollingUpdate.maxSurge }} + maxUnavailable: {{ $.Values.strategy.rollingUpdate.maxUnavailable }} + {{- end }} template: metadata: labels: @@ -19,13 +26,6 @@ spec: annotations: confighash: secret-{{ .Values.dendrite_config | toYaml | sha256sum | trunc 32 }} spec: - strategy: - type: {{ $.Values.strategy.type }} - {{- if eq $.Values.strategy.type "RollingUpdate" }} - rollingUpdate: - maxSurge: {{ $.Values.strategy.rollingUpdate.maxSurge }} - maxUnavailable: {{ $.Values.strategy.rollingUpdate.maxUnavailable }} - {{- end }} volumes: - name: {{ include "dendrite.fullname" . }}-conf-vol secret: @@ -47,6 +47,9 @@ spec: - name: {{ include "dendrite.fullname" . }}-search persistentVolumeClaim: claimName: {{ default (print ( include "dendrite.fullname" . ) "-search-pvc") $.Values.persistence.search.existingClaim | quote }} + {{- with .Values.extraVolumes }} + {{ . | toYaml | nindent 6 }} + {{- end }} containers: - name: {{ .Chart.Name }} {{- include "image.name" . | nindent 8 }} @@ -80,6 +83,9 @@ spec: name: {{ include "dendrite.fullname" . }}-jetstream - mountPath: {{ .Values.dendrite_config.sync_api.search.index_path }} name: {{ include "dendrite.fullname" . }}-search + {{- with .Values.extraVolumeMounts }} + {{ . | toYaml | nindent 8 }} + {{- end }} livenessProbe: initialDelaySeconds: 10 periodSeconds: 10 diff --git a/helm/dendrite/values.yaml b/helm/dendrite/values.yaml index d6be2cdbbb..b0e8fc8a83 100644 --- a/helm/dendrite/values.yaml +++ b/helm/dendrite/values.yaml @@ -43,6 +43,20 @@ persistence: # -- PVC Storage Request for the search volume capacity: "1Gi" +# Add additional volumes to the Dendrite Pod +extraVolumes: [] +# ex. +# - name: extra-config +# secret: +# secretName: extra-config + + +# Configure additional mount points volumes in the Dendrite Pod +extraVolumeMounts: [] +# ex. +# - mountPath: /etc/dendrite/extra-config +# name: extra-config + strategy: # -- Strategy to use for rolling updates (e.g. Recreate, RollingUpdate) # If you are using ReadWriteOnce volumes, you should probably use Recreate From 8223e1f2e150e94314c7a68fcd5ba45128d29e6e Mon Sep 17 00:00:00 2001 From: genofire Date: Mon, 3 Apr 2023 09:04:59 +0200 Subject: [PATCH 24/42] fix(helm): improve documentation and grafana dashboard (#2992) ### Pull Request Checklist * [x] I have added Go unit tests or [Complement integration tests](https://github.com/matrix-org/complement) for this PR _or_ I have justified why this PR doesn't need tests * [x] Pull request includes a [sign off below using a legally identifiable name](https://matrix-org.github.io/dendrite/development/contributing#sign-off) _or_ I have already signed off privately --- * **docs**: enabling of metrics in there config is needed * **dashboard**: since the imported dashboard was created: * many metrics was dropped by dendrite (mainly #2967) * grafana has new version of diagrams ... --------- Co-authored-by: Till <2353100+S7evinK@users.noreply.github.com> [skip ci] --- helm/dendrite/.helm-docs/monitoring.gotmpl | 9 +- helm/dendrite/README.md | 11 +- .../grafana_dashboards/dendrite-rev1.json | 1119 ----------------- .../grafana_dashboards/dendrite-rev2.json | 479 +++++++ .../grafana_dashboards/dendrite-rev2.png | Bin 0 -> 128391 bytes 5 files changed, 494 insertions(+), 1124 deletions(-) delete mode 100644 helm/dendrite/grafana_dashboards/dendrite-rev1.json create mode 100644 helm/dendrite/grafana_dashboards/dendrite-rev2.json create mode 100644 helm/dendrite/grafana_dashboards/dendrite-rev2.png diff --git a/helm/dendrite/.helm-docs/monitoring.gotmpl b/helm/dendrite/.helm-docs/monitoring.gotmpl index 3618a1c1a1..eaa336ebd9 100644 --- a/helm/dendrite/.helm-docs/monitoring.gotmpl +++ b/helm/dendrite/.helm-docs/monitoring.gotmpl @@ -1,10 +1,15 @@ {{ define "chart.monitoringSection" }} ## Monitoring -[![Grafana Dashboard](https://grafana.com/api/dashboards/13916/images/9894/image)](https://grafana.com/grafana/dashboards/13916-dendrite/) +![Grafana Dashboard](grafana_dashboards/dendrite-rev2.png) * Works well with [Prometheus Operator](https://prometheus-operator.dev/) ([Helmchart](https://artifacthub.io/packages/helm/prometheus-community/kube-prometheus-stack)) and their setup of [Grafana](https://grafana.com/grafana/), by enabling the following values: ```yaml +dendrite_config: + global: + metrics: + enabled: true + prometheus: servicemonitor: enabled: true @@ -19,4 +24,4 @@ grafana: enabled: true # will deploy default dashboards ``` PS: The label `release=kube-prometheus-stack` is setup with the helmchart of the Prometheus Operator. For Grafana Dashboards it may be necessary to enable scanning in the correct namespaces (or ALL), enabled by `sidecar.dashboards.searchNamespace` in [Helmchart of grafana](https://artifacthub.io/packages/helm/grafana/grafana) (which is part of PrometheusOperator, so `grafana.sidecar.dashboards.searchNamespace`) -{{ end }} \ No newline at end of file +{{ end }} diff --git a/helm/dendrite/README.md b/helm/dendrite/README.md index c3833edfba..90e31e040a 100644 --- a/helm/dendrite/README.md +++ b/helm/dendrite/README.md @@ -1,6 +1,7 @@ + # dendrite -![Version: 0.12.0](https://img.shields.io/badge/Version-0.12.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.12.0](https://img.shields.io/badge/AppVersion-0.12.0-informational?style=flat-square) +![Version: 0.12.1](https://img.shields.io/badge/Version-0.12.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.12.0](https://img.shields.io/badge/AppVersion-0.12.0-informational?style=flat-square) Dendrite Matrix Homeserver Status: **NOT PRODUCTION READY** @@ -157,10 +158,15 @@ Create a folder `appservices` and place your configurations in there. The confi ## Monitoring -[![Grafana Dashboard](https://grafana.com/api/dashboards/13916/images/9894/image)](https://grafana.com/grafana/dashboards/13916-dendrite/) +![Grafana Dashboard](grafana_dashboards/dendrite-rev2.png) * Works well with [Prometheus Operator](https://prometheus-operator.dev/) ([Helmchart](https://artifacthub.io/packages/helm/prometheus-community/kube-prometheus-stack)) and their setup of [Grafana](https://grafana.com/grafana/), by enabling the following values: ```yaml +dendrite_config: + global: + metrics: + enabled: true + prometheus: servicemonitor: enabled: true @@ -175,4 +181,3 @@ grafana: enabled: true # will deploy default dashboards ``` PS: The label `release=kube-prometheus-stack` is setup with the helmchart of the Prometheus Operator. For Grafana Dashboards it may be necessary to enable scanning in the correct namespaces (or ALL), enabled by `sidecar.dashboards.searchNamespace` in [Helmchart of grafana](https://artifacthub.io/packages/helm/grafana/grafana) (which is part of PrometheusOperator, so `grafana.sidecar.dashboards.searchNamespace`) - diff --git a/helm/dendrite/grafana_dashboards/dendrite-rev1.json b/helm/dendrite/grafana_dashboards/dendrite-rev1.json deleted file mode 100644 index 206e8af878..0000000000 --- a/helm/dendrite/grafana_dashboards/dendrite-rev1.json +++ /dev/null @@ -1,1119 +0,0 @@ -{ - "__inputs": [ - { - "name": "DS_INFLUXDB_DOMOTICA", - "label": "", - "description": "", - "type": "datasource", - "pluginId": "influxdb", - "pluginName": "InfluxDB" - }, - { - "name": "DS_PROMETHEUS", - "label": "Prometheus", - "description": "", - "type": "datasource", - "pluginId": "prometheus", - "pluginName": "Prometheus" - } - ], - "__requires": [ - { - "type": "grafana", - "id": "grafana", - "name": "Grafana", - "version": "7.4.2" - }, - { - "type": "panel", - "id": "graph", - "name": "Graph", - "version": "" - }, - { - "type": "panel", - "id": "heatmap", - "name": "Heatmap", - "version": "" - }, - { - "type": "datasource", - "id": "influxdb", - "name": "InfluxDB", - "version": "1.0.0" - }, - { - "type": "datasource", - "id": "prometheus", - "name": "Prometheus", - "version": "1.0.0" - }, - { - "type": "panel", - "id": "stat", - "name": "Stat", - "version": "" - } - ], - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "description": "Dendrite dashboard from https://github.com/matrix-org/dendrite/", - "editable": true, - "gnetId": 13916, - "graphTooltip": 0, - "id": null, - "iteration": 1613683251329, - "links": [], - "panels": [ - { - "collapsed": false, - "datasource": "${DS_INFLUXDB_DOMOTICA}", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 4, - "panels": [], - "title": "Overview", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 5, - "w": 10, - "x": 0, - "y": 1 - }, - "hiddenSeries": false, - "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.4.2", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "rate(process_cpu_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{job}}-{{index}} ", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "CPU usage", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": null, - "format": "percentunit", - "label": null, - "logBase": 1, - "max": "1", - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "datasource": "${DS_PROMETHEUS}", - "description": "Total number of registered users", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": {}, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 4, - "x": 10, - "y": 1 - }, - "id": 20, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "7.4.2", - "targets": [ - { - "exemplar": false, - "expr": "dendrite_clientapi_reg_users_total", - "instant": false, - "interval": "", - "legendFormat": "Users", - "refId": "A" - } - ], - "title": "Registerd Users", - "type": "stat" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "description": "The number of sync requests that are active right now and are waiting to be woken by a notifier", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 5, - "w": 10, - "x": 14, - "y": 1 - }, - "hiddenSeries": false, - "id": 6, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.4.2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(dendrite_syncapi_active_sync_requests{instance=\"$instance\"}[$bucket_size]))without (job,index)", - "hide": false, - "interval": "", - "legendFormat": "active", - "refId": "A" - }, - { - "expr": "sum(rate(dendrite_syncapi_waiting_sync_requests{instance=\"$instance\"}[$bucket_size]))without (job,index)", - "hide": false, - "interval": "", - "legendFormat": "waiting", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Sync API", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:232", - "format": "hertz", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:233", - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "cards": { - "cardPadding": null, - "cardRound": null - }, - "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", - "colorScheme": "interpolateOranges", - "exponent": 0.5, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", - "datasource": "${DS_PROMETHEUS}", - "description": "How long it takes to build and submit a new event from the client API to the roomserver", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 24, - "x": 0, - "y": 6 - }, - "heatmap": {}, - "hideZeroBuckets": false, - "highlightCards": true, - "id": 24, - "legend": { - "show": false - }, - "pluginVersion": "7.4.2", - "reverseYBuckets": false, - "targets": [ - { - "expr": "dendrite_clientapi_sendevent_duration_millis_bucket{action=\"build\",instance=\"$instance\"}", - "interval": "", - "legendFormat": "{{le}}", - "refId": "A" - } - ], - "title": "Sendevent Duration", - "tooltip": { - "show": true, - "showHistogram": false - }, - "type": "heatmap", - "xAxis": { - "show": true - }, - "xBucketNumber": null, - "xBucketSize": null, - "yAxis": { - "decimals": null, - "format": "s", - "logBase": 1, - "max": null, - "min": "0", - "show": true, - "splitFactor": null - }, - "yBucketBound": "auto", - "yBucketNumber": null, - "yBucketSize": null - }, - { - "collapsed": false, - "datasource": "${DS_INFLUXDB_DOMOTICA}", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 11 - }, - "id": 8, - "panels": [], - "title": "Federation", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "description": "Collection of queues for sending transactions to other matrix servers", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 24, - "x": 0, - "y": 12 - }, - "hiddenSeries": false, - "id": 10, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.4.2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "dendrite_federationsender_destination_queues_running", - "interval": "", - "legendFormat": "Queue Running", - "refId": "A" - }, - { - "expr": "dendrite_federationsender_destination_queues_total", - "hide": false, - "interval": "", - "legendFormat": "Queue Total", - "refId": "B" - }, - { - "expr": "dendrite_federationsender_destination_queues_backing_off", - "hide": false, - "interval": "", - "legendFormat": "Backing Off", - "refId": "C" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Federation Sender Destination", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:443", - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "$$hashKey": "object:444", - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "collapsed": false, - "datasource": "${DS_INFLUXDB_DOMOTICA}", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 18 - }, - "id": 26, - "panels": [], - "title": "Rooms", - "type": "row" - }, - { - "cards": { - "cardPadding": null, - "cardRound": null - }, - "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", - "colorScheme": "interpolateOranges", - "exponent": 0.5, - "mode": "spectrum" - }, - "dataFormat": "timeseries", - "datasource": "${DS_PROMETHEUS}", - "description": "How long it takes the roomserver to process an event", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 24, - "x": 0, - "y": 19 - }, - "heatmap": {}, - "hideZeroBuckets": false, - "highlightCards": true, - "id": 28, - "legend": { - "show": false - }, - "pluginVersion": "7.4.2", - "reverseYBuckets": false, - "targets": [ - { - "expr": "sum(rate(dendrite_roomserver_processroomevent_duration_millis_bucket{instance=\"$instance\"}[$bucket_size])) by (le)", - "interval": "", - "legendFormat": "{{le}}", - "refId": "A" - } - ], - "title": "Room Event Processing", - "tooltip": { - "show": true, - "showHistogram": false - }, - "type": "heatmap", - "xAxis": { - "show": true - }, - "xBucketNumber": null, - "xBucketSize": null, - "yAxis": { - "decimals": null, - "format": "s", - "logBase": 1, - "max": null, - "min": null, - "show": true, - "splitFactor": null - }, - "yBucketBound": "auto", - "yBucketNumber": null, - "yBucketSize": null - }, - { - "collapsed": false, - "datasource": "${DS_INFLUXDB_DOMOTICA}", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 26 - }, - "id": 12, - "panels": [], - "title": "Caches", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 8, - "x": 0, - "y": 27 - }, - "hiddenSeries": false, - "id": 14, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.4.2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "dendrite_caching_in_memory_lru_server_key", - "interval": "", - "legendFormat": "Server keys", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Server Keys", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:667", - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "$$hashKey": "object:668", - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 8, - "x": 8, - "y": 27 - }, - "hiddenSeries": false, - "id": 16, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.4.2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "dendrite_caching_in_memory_lru_federation_event", - "interval": "", - "legendFormat": "Federation Event", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Federation Events", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:784", - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "$$hashKey": "object:785", - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 8, - "x": 16, - "y": 27 - }, - "hiddenSeries": false, - "id": 18, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.4.2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "dendrite_caching_in_memory_lru_roomserver_room_ids", - "interval": "", - "legendFormat": "Room IDs", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Room IDs", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:898", - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "$$hashKey": "object:899", - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - } - ], - "refresh": "10s", - "schemaVersion": 27, - "style": "dark", - "tags": [ - "matrix", - "dendrite" - ], - "templating": { - "list": [ - { - "current": { - "selected": false, - "text": "Prometheus", - "value": "Prometheus" - }, - "description": null, - "error": null, - "hide": 0, - "includeAll": false, - "label": null, - "multi": false, - "name": "datasource", - "options": [], - "query": "prometheus", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "type": "datasource" - }, - { - "auto": true, - "auto_count": 100, - "auto_min": "30s", - "current": { - "selected": false, - "text": "auto", - "value": "$__auto_interval_bucket_size" - }, - "description": null, - "error": null, - "hide": 0, - "label": "Bucket Size", - "name": "bucket_size", - "options": [ - { - "selected": true, - "text": "auto", - "value": "$__auto_interval_bucket_size" - }, - { - "selected": false, - "text": "30s", - "value": "30s" - }, - { - "selected": false, - "text": "1m", - "value": "1m" - }, - { - "selected": false, - "text": "2m", - "value": "2m" - }, - { - "selected": false, - "text": "5m", - "value": "5m" - }, - { - "selected": false, - "text": "10m", - "value": "10m" - }, - { - "selected": false, - "text": "15m", - "value": "15m" - } - ], - "query": "30s,1m,2m,5m,10m,15m", - "queryValue": "", - "refresh": 2, - "skipUrlSync": false, - "type": "interval" - }, - { - "allValue": null, - "current": {}, - "datasource": "${DS_PROMETHEUS}", - "definition": "label_values(dendrite_caching_in_memory_lru_roominfo, instance)", - "description": null, - "error": null, - "hide": 0, - "includeAll": false, - "label": null, - "multi": false, - "name": "instance", - "options": [], - "query": { - "query": "label_values(dendrite_caching_in_memory_lru_roominfo, instance)", - "refId": "StandardVariableQuery" - }, - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": {}, - "datasource": "${DS_PROMETHEUS}", - "definition": "label_values(dendrite_caching_in_memory_lru_roominfo, job)", - "description": null, - "error": null, - "hide": 0, - "includeAll": true, - "label": "Job", - "multi": true, - "name": "job", - "options": [], - "query": { - "query": "label_values(dendrite_caching_in_memory_lru_roominfo, job)", - "refId": "StandardVariableQuery" - }, - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": ".*", - "current": {}, - "datasource": "${DS_PROMETHEUS}", - "definition": "label_values(dendrite_caching_in_memory_lru_roominfo, index)", - "description": null, - "error": null, - "hide": 0, - "includeAll": true, - "label": null, - "multi": true, - "name": "index", - "options": [], - "query": { - "query": "label_values(dendrite_caching_in_memory_lru_roominfo, index)", - "refId": "StandardVariableQuery" - }, - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 3, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-3h", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "Dendrite", - "uid": "RoRt1jEGz", - "version": 8 -} \ No newline at end of file diff --git a/helm/dendrite/grafana_dashboards/dendrite-rev2.json b/helm/dendrite/grafana_dashboards/dendrite-rev2.json new file mode 100644 index 0000000000..817f950b34 --- /dev/null +++ b/helm/dendrite/grafana_dashboards/dendrite-rev2.json @@ -0,0 +1,479 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "Dendrite dashboard from https://github.com/matrix-org/dendrite/", + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": 13916, + "graphTooltip": 0, + "id": 60, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 4, + "panels": [], + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "refId": "A" + } + ], + "title": "Overview", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Total number of registered users", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 7, + "x": 0, + "y": 1 + }, + "id": 20, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "9.3.6", + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(dendrite_clientapi_reg_users_total{namespace=~\"$namespace\",service=~\"$service\"}) by (namespace,service)", + "instant": false, + "interval": "", + "legendFormat": "{{namespace}}: {{service}}", + "refId": "A" + } + ], + "title": "Registerd Users", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "The number of sync requests that are active right now and are waiting to be woken by a notifier", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 17, + "x": 7, + "y": 1 + }, + "id": 6, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.3.6", + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(rate(dendrite_syncapi_active_sync_requests{namespace=~\"$namespace\",service=~\"$service\"}[$__rate_interval]))by (namspace,service)", + "hide": false, + "interval": "", + "legendFormat": "active: {{namspace}} - {{service}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(rate(dendrite_syncapi_waiting_sync_requests{namespace=~\"$namespace\",service=~\"$service\"}[$__rate_interval]))by (namespace,service)", + "hide": false, + "interval": "", + "legendFormat": "waiting: {{namspace}} - {{service}}", + "range": true, + "refId": "B" + } + ], + "title": "Sync API", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 8, + "panels": [], + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "refId": "A" + } + ], + "title": "Federation", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Collection of queues for sending transactions to other matrix servers", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 10, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.3.6", + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "dendrite_federationapi_destination_queues_running{namespace=~\"$namespace\",service=~\"$service\"}", + "interval": "", + "legendFormat": "Queue Running: {{namespace}}-{{service}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "dendrite_federationapi_destination_queues_total{namespace=~\"$namespace\",service=~\"$service\"}", + "hide": false, + "interval": "", + "legendFormat": "Queue Total: {{namespace}}-{{service}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "dendrite_federationapi_destination_queues_backing_off{namespace=~\"$namespace\",service=~\"$service\"}", + "hide": false, + "interval": "", + "legendFormat": "Backing Off: {{namespace}}-{{service}}", + "range": true, + "refId": "C" + } + ], + "title": "Federation Sender Destination", + "type": "timeseries" + } + ], + "refresh": "10s", + "schemaVersion": 37, + "style": "dark", + "tags": [ + "matrix", + "dendrite" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "includeAll": false, + "label": "datasource", + "multi": false, + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(dendrite_syncapi_active_sync_requests, namespace)", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "namespace", + "options": [], + "query": { + "query": "label_values(dendrite_syncapi_active_sync_requests, namespace)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(dendrite_syncapi_active_sync_requests{namespace=~\"$namespace\"}, service)", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "service", + "options": [], + "query": { + "query": "label_values(dendrite_syncapi_active_sync_requests{namespace=~\"$namespace\"}, service)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-3h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Dendrite", + "uid": "RoRt1jEGz", + "version": 1, + "weekStart": "" +} diff --git a/helm/dendrite/grafana_dashboards/dendrite-rev2.png b/helm/dendrite/grafana_dashboards/dendrite-rev2.png new file mode 100644 index 0000000000000000000000000000000000000000..179f4b5a187d2f2a61c2743ed93790658e98d8f7 GIT binary patch literal 128391 zcmbSzbySsGyDy4>N-EtTsdP6;i?q@y-QBfNLAp`8n?*NBi8RtkF1owB)}5U1?0wE1 zQiL!9kJy@e(=91 zPGT}DPr-lQPmP1XV?t+14d;(`rp|5#jwT3ZwstlqOio6QCMLE{=622ph^@llNwg17 z5_dE)aJI0srBJc3F+or_F{WT;qmVanq+n-d=b&Ka<7emKXJxJVmG>0^fdWBBLR7^) zWp~c)lZxu~;n6`eM{Iu%&xfriVnn23s92sT6<nBIkb-Bii*$u%BzW4STbb(Ow zf?|5bgK0M3{fI+zda~aHp(mi)w9S)1%uX z#w~NRnn(iM9mdzSQZfA6&{fCcgi$D=<~3TyYF1WmD|C=&-9O*hoZQ*ag!8(-3Hho$ zP@!)jz=w+wh#orqwW6|8v>Prf^PkJI31|#NQA@_n9vPj{wZOq|wrXd_IxS7;&%@hP zUZD^93Vuw}*{N;=Qxl;Im-1^H2oE_vE~@1sm@U^B;AIRa=0zQOIXfrBM#P}iNPOlv zvP0}%JXY6hL|VOfQOCu3I%Q$+)cyrAyLU=V$cnc_EmW$g)6!zrqj1mP`C&aB+Q%$i zruNP!_NB64zi&4cCY~wHET@-R2{KlfxNB7N`a&&)g9_OgR``yXSAr8P^F6cWkjsJs)w zKZ%YVYOW$x{?|RiQm6j^yX)vM-{R;~Pee*e%Cq~D^75#xEHMNzF|m=IGnbhuYuCz3 zcG&KOkQk?mgaozQ9AEmLKyA6_S;)lxAUYL2Jpn#GK0C)ywW~T5}?c zZ3WCm@b2_62JODxq0{+}K$NcQDIec-33m?>D@DY!mHfl`8~7`h z1F@phOvn?Xr(=J!D>y9 zD@?73hMM~OeDn1SvibHaA_PY#r+4p#dwOH(=wH9K?bHh){*hiLGBQ#jA=S0Gx3<^R zT%VI59*7Z@#TU`KIu^sEWq)>@%i;$L&{wron+d21BCByT%re97LPn<2Kn zH#mjgB_<@~SZ%+i{v|oNzHO2N!=p!gL_Q0Beb3{{blyvK6`F2NOAdS|%<8tRIbONu z`mXC}sPK*;vb9wNK`%{JWKdiNg)c5`+j^|fNgWv@sJ$jFEIL3*^+#`Po<19aAuB=s z-Fd_0>1CElM<;r2R@T?*>hDFh&Gt)zfiV~v^v~z#g(alVUUAoAI<3zR96;c#8$%nl zZzIdn)r#5{L=fyV#w}SB-)4^-c6`m>9L~hX6yi*eQ6t)%XEP(KsAzsCD{C;a{HK4CbDwJSW)-=aQPwWxLD^5}G8-?3UmQK_Kqu-;a$z9m?6Q^F-{0S?=07d-3jJM|R2uNq#a(pyU5PeF@IV?NVbwLF?R% zE{+%{2M-U=MkOk^NKH)(p%P1S*>N@gT5c8iz28%e=Sz!iJ_$^w%gE?7cN%Z#6i5(% zeX=+D+r#GW?(Vq5$Lbw*wJxl+yZ+fn@$`V z)2wfm(0aB$BBwn#98&6FL^}#TK<(~ONIHVd78)s`zdnk_6$@zZU6Z-P9nLaOj|h*0T53ZGlH+jy+=B?St`eo<`n?%tlb@sK%K z4}*z*m`1ZHO}8Sa-LP`LQ>JWusq@}Tg-PZm@2hb-m#*IxY|VtM6d3yCBE{Sc8-pel zCOcA~JSElG+5Bkx^tOyAD%L52Ye$Xi`aUg9&v|RMgwRtB#!~uuyHpnjs_WL6-B0V} zOz{=AWO?;|f?;z`cMp%1gN?|@NW115dCSc+Pj4UZqmz?v7_6HQcKQ(tU0zxWPEQvB zy~f?slYh*4$B32S@a(L}c1Jb8L?OQ&lnHRVaU|Q?+Yv=Xe7$|{hIi1_cgqxAe1!y| z*PuX#r>DbBW+wNgeK>DEq$t4}TT}4+-}7$-+Iu#7_jG9QCC??CD8!3I*5)XX@_)1_ zHoh5T8bpn6YC_E+mwnjz;QbJOS=KDCto)gCI%uEluH$Od2~2JaSElOv#GQC~Id;L9 z;vMw$R#2@>^VwTTNu=c3;{2?p6#3-ID$jxEXr4kVU2^7YY5F9`H7lueUyxynXt) z(-3?`q0e~xgaji`zWJG=Vwu|Z(OLuHEHu)FE_J0r78f5MoETT(n`J5-NfU7R2KWa< z!?)VABL&+|3#)>H_rEnZdZluM%@sy2LWt?6c)mN=d$h9UygQtDWJEXVHDFKGFtLi5 zJmA{7vzj+*ok&@-X01j$J#Dr%)+mDP_oc6IhVYSR^OrAv0rvOa%nc0(c0hGxVP@h z`OY9$!N^GS%t8inf3yd4 zlgJ-aR#E9$XCmI5bVK;^MYKPGuSMw{&j}D+k33QpEog8X*qEx|R7eq=FSya`Te3ggGJ_l4 zUiS8<-Y}@$m6IfwY=7QfUmtA86jA_{7@XmD%NoafZ;U@J`u(H1p`W3mL+YsU(TRy| zlMCK$yReOy1O)z2=<3GC(bWcdg{S=9*Mf`w#&m}j+NGVHKydpR7-Bm*5M8d%Z=G1i zO0?Y|8qHrHE^cFENZ_EQ_h*gBVI`_@+LSTMxfsjc7T8# zF|R%A#rzEd=$B$es7ryA!NBt>-oDmBO2~F2HXnl}iL{n|tt4%a}^ z&`vpkz9l6kT?W$;WMqP=X;e*UpCBU}RZSOIi)*HWQ}zM}=l7QmAM3f=!UYm@NBw@%BYfjYgCIv-I{VPfu5ljEE%vpy0Gsi6n*&I1zX%*W4jD_Osdro*@5s28bx5Dqzc_D(Sm?GDh0E(fJbH^xWcYR_{ zcE3A|w@Q_uRZRObZEd{Lb%uktj!g^;{DO#bvU5YeUMC%sA{x55sC0k3(Ymz1-&VVq zEF_b&vTLzd6=y zZ*PwsdA7m_5e$!siLugpKO$U=Xkb8k=q?BTY!)=eo@DsL~8r0=9uYI$H`MDH+pBP)+c0wUf z!Hc;ItmHcDjLOQ#lFA7}Ud#`?qrqv~`)&aL`*)m?d^g`b`D9JIh1uEJM1B+6?#O%x zPMXA&6epJ%@5>P{M$Ky`OrcZluYO`4S7k%Wdae#i3;Av$X}=i#JGy2Evegv$m0f!s zaPMfk`}1Tgt=O-wJW@)?%=MwgO<_jeqwnNpMay_tGvE~aO-8~C-Zx0#2_=7FGvKg3 zv&Q!aXv?r-abu@D=!Fp=B#5Eozca;ArU17#tx8m_gLbO)Xw18?x4*Vn$5COV@&!VaK6nq->|u!KI!}O2T*zEz0q*6iM`JTHb5mH zD^dINy_aK?j7-$P;1R+Lp;y6SL55owp~E|8gN%BqYJu89E6$&tHYznMPqjDTHCjxN z7Kkd>$M>Z=L1S|RsSR#}-RA5D!#i6e&2#`ySvGRYOHu~F!zx%K6VdYUS7V$A^94gkDp%@--j7N zEOmRRj1!Mz#Fo4TE>I%!UCZ@))y{l#&$z0hwYA-c4>49ucF3Efxy0Pb;T&cP`jbwa zDk{ckV2!?iUq!A_l#!t|Sk=hNno(dihJR^xiMzZz^|>pglExgXv-8c$_N;K(F@66h zNf@CC!0V0HK#btB6Sr_EB{eTj)wG1E3)YZrozrp$+fV0xsSO2PUN>e`fPD88e0Ye$ z5C?dR|kBXw0C?CkB-(Z zT5qtCkdOqgr|t$5BWkSFk5qe!eYdWMg@Q#kgodh`T+r z8>%wuR?FDF*;?N2D~L8C;O9&`+nK}yo!q8wUKDgGOc6(x++02KLbSBjlr7Q%dV~DYHaDba18gl&Hnwz1pP})j0&v!MNZg|UC@Md;wkT9Rz zS0`Mp$msGg`gBL&^Bu}L_omPGR3Wwv9%o+pWI|4cVlRuUS3m^z6n3#$iaD0}lpUM{^j;E|Y_<9)$ zmY&@~*|}9GLoL=$P#W1NXK+7DYY>s z!GN1=i^aDzy}t~#ETC;KZA0^mi*b?%0>BY>oAY2PEiFxHjS2-M3Gur71<_2}H+`S? z$_{L{!2D$$u-r`@B+n+705(aLh5;a+8V>Azs z4b7%iVnF2p0%*9K5FjMs2?HAw3vVWkY|bktE^WELE0Ns;S+~(28;77Dh>gK}UGvkq zjnft(Vds2vV<{w%HgCjI38!W-CN0S-Kgom(U@caHS&fSo%a};&rKSD0<_qCp{th4C zDJqs!6&DIqwoy`t5y{@1?}2(!adPfpM2AH@7U&mvF3Xw|okT-WX}C7ZkCbcqxBoM_ zK-wTsbHeA9+~i9KOsVc-NKJ{_$5n?4Khp8K+l$;?V`Lb04-O(K*V)b;ogbU7^_fiV zVVGH3pyT7KU!M`bQyAoP-JR943LiUKu~1R@K9=p}e^Q%-)3_MLx?uyh0ZYqef_yTB zy45po*0RR13g+N+LBYw{KTQt$7Eqf*8yitP#yNO^Y6GxL&`|4T58=~7Pc&40=3D=G zJ@4@QJCyO~0oHRr3ny-DxRZMagrrqe(9+UCX~kPyUVcAWx`czXT4g%Wr_~nH-;Wk1 zMFn={APgL_kh(ga1?Fq$#0M%MZKpLi5^$J4o@@8|`#j)#e0;Pw^;QxhWk?pQ&9*#= zK-xMZnp?9ayE+i-faJ_-$%u=K>wUFGS99&L-fuCe&#PE>NM_DSWQH|lUTnNUBg?q! zDu8l+U;-qqYP*FOpD{@JmjUmJ+?fQtoXaLsk5)`yQ?s|N<8IRXI5>DqGG9UW(%|e^ ztFR7qES8qrgv(21>M*j&#ZNtn{b&=#x}DX9JjyCMzd~7rWj#ILr)HVsg?emGO8%etMG%caZM)`>Ec)B?3K3R|QBGm_EtN2QIa@2; z#MTyr+~@RUDoy#R!yZs$t;X|y&i>q$4Tb6@4Vc%#w(v68U!RZ4B!m(^Z59weNE)>a zwcseNy`NdKFZUY402^J?N_$}9CEf&?y1Gzs;CbwK6|Q7LT+JV&trjPdphgpiOZ}-X zCJ!3$d$Qm`zLu4drwX{VePUf)&gxz&2Q)=oRaI^s>QoPuc)*qc&8~2r zdjrHj0C6j(Z3Yiw=WJWIF9!^394QY3!HowE%%~#4funw0GX_!^6Z4XN$kdQKxg2Km zJuVLHm#29ML^J<^bycEx0fNor}IX$G)=H zTC9v`IMYX3AzirloK%}P#rx@)i|Mzws>sLJ;XCgLxZ?P#HjeL(4}u~>OzdavBS;m^ z)}Vm8lAvd*=@0$&#`u`uUvIe73$S&$xx@ISo-&TNw|B4AuK=V0nKB+M?6FYbDR9!$ z+ZW#6dNbm22&)%(jium7K0+cnPd7CZfWq}$$Qlv~$;H&vXs?hwyl;C^3ZOHKR{7X; zkySP~rU7VVgJ9zvwQsM51t|hLsQ(gk&uomg*Wb3-^jSbo9$E9Qnjsf?ZKvUSBh5k< z)CGkFBQSgZD<8T+0kiWpxcJSnd~)%?E&)HxY^eyzqVWo!ZZ#Z9(d@$1Z_8lKA|qM$ zW{@hKduKe!;iLUOfSbi?HDNO=bfG6j7iM&Id;n-Rs${2R&8uV}pfc)QzHa?_Au8++ z=WyMh?b;q^03AzALBX|Ta;9tn0b&2AlQ3|_TIYqq?V7yym1hc*9oOmDt{&ft$Y0sJa?n0wpKfB+;UB6RCMFRd%w2NbGCFhI^0TJ5xsvJ-vUW^ z)0rQ%<$Q%y^NP7ArCmxw4i{r<{8Z9mt`Vqz`rhv}5I_yt6AzCpuTg9CDtsQuCTKRH zonNz5B0m-(`aLwlN*5}>l>o<`0Vjla@GnrTDUD zh2xgV!N-l`VFPJpDD=kNQ_aM6O}Adjs`!UPo@Gq;(8aN6-XFHRRTmprOJeX!{sZ;J zk)&i0h#k=8=I*W33v3q!wnv8LMf%kVBwWvqg5iddL7l0{&3m525^}(`Fow%8-$ZHg zPZl6vw<5z_{?kYXk5Y}Ffls4tO>qRNKs2i(BT#_C2QeyOdj#Q9r==wyPeD=ooZa%d z>BHRxbcl|ICb6J^;FXsYHt|?8&awYG#)K z-lXL^qsQ)2G~FC_4?rgtwXk?C9YK9G4OLYD_IDSr((6K@kGX1pZ>pkNs+Hxr zY8!b`=S&89h`NjOSx#Mz8zQXZ)=~=8MeQ3-&yhFg&DleqB4>zZk$%O z$bQ6B%h*rrjosK#RAtRw59Zq%{-jiC$13C-0Y!m+O$~3q1K>eu zrod&94sI49yE`xNt%2--j`R%)xFZms1*)2vz3Il->QkJ@fT`v>)omL>b+5LQhlq3( zy9Nf(d3Z2^iIGwT+i)B-;7$g_-Fcua)0Dmaj81@iBCLq{0>a$|Y(mhpUH}yfxC3Lr zg|cll!C+AL8LC(bCs&-@I9!tGH|4;llUwTOH7eGacKm|%`Ri9f(A~Dk=COgbBxOW1 zJuZ8|BAq`r((I*pe0)-xFg^CcGDN7caeS_D%@R0|3(R^`mXZ!OYQz|U&4bKITpP(r z#YgXrXaI)#?z;CFT$+TOGa~@-*z+YQ`aUNfeuxjkY>76HL2)s)Fa!rQ_PS{x^#j~& zkU6Q2UCqY@(4v_H4sf9>`g*M){T73bG1OuLwX}x$dTw?Ocm{Oz z^p{CrDY--&Q1J(=eBoqLVFdC3nVlYpJZ+x;MVEdHB-RghcP|6yB03E-K^lSQ{%De6 zFl3S!Y*oonyh`&CF|f>Rj;jQ=jb|8;iw^SMenD~1>&`Vs=}5Bm&5`z|tBE2B2?=(v zsX^1(2%m%xh+{6uT39^m>{R8V1k!iEW`5S|DN{w_aoxJ-AkA&ns0A#@Grxg%D1?d!UUB$&=JPRPEA%Mo2=PEB_arUSw zt)w^J^{F*}2XM6rxveFdBYhXj$g;!(BLt5+?08dV zo!g480X!`?Xy>PuPwF~8@O=V(wE{F7V9%TMM7g)JFq#4CGNhA`%>PV+E96~!8prDLazT|h^4U@`t(c9#vZlq}Pys}YK zGmSl$$sWm=B<=yN$v~OXwULyCv z1z$;Ifb*xy#o^Pg$mIUgIaul+HGttbi3VJd7@t3X&YC>n|1IFb@CRx+6gmo&J_L4- zq7Y@R58ygKUdlmfGyp#>X zqCKxxjZ*c1E!B+<-dsD?L+$`rcM__qKV49}m+X1f=>fX>)!iL`Mm+J5NC7g>?e)F% zQ`DW$`GItu@qdw;hyvU7YQ7x=JBNgX%ve)Af%dUINNwRQ3t?M4fEo>%ZYtXhsJ$IG z=RDrPzn4&(=sx!MMjkmF@O?b`7NWobxgzN_c@H$fH9L1TR$*$>$2LJ^R4}6WLO9n^BN{{=7R(79j zMm?`Rng_vQpvu;4a(~Zyp(Xb^!{d8(H4aE`1p%T8q~y`j_fx)~SRNYTb+x8E5gJ=E z6A->1aDVfB&BV{G>lC5nU_cRRf2xN*5&!ADB?Y1oaLC30?UiT|v`^?{UtAdnunsDq zD1{~b{{5KWWn0YN{$1=iO`+yC4%vLR3|SearDc~~I(LzbAEgr4W)H7Z2&H&{zN_id zW$-Cxl8{jY+5sf6BW$+ocEd766stVC+04P0a_H z5eogFtUT!d>>~k)5TTHzs9P0@Qu-gBpD#@$Z`cDy@Suhx;7rO6-h^P7Yr}Tbw%0_2DKhQ^OtAH=NXSe*IiT*7lfC)nXB zH5v6UoQ3S9lDy}C3vLnE$vr~p;_ofT9fN@As0x^_!$h(X$rKtcg&u|F|yVX!*q1%8!06W1O+$eysX3xBCH8`7!hiU2tQsugV3 z#IX1ho5@k&r+Zv^`1|`WZ$rzf+D*_Ul{XiBu`(&+O^JAUzdL^plVVc0(*l>DuRXah zm+XlR46*jHu~-Gt#;91YL60VI{v}R>B+4e8Hit!l{&RSGm{VE#o3D3TqeNHZ`aNio zpwz%WyE^aB<|>PN0?9~21LeJJvai=AFoZrM0%@kNFSP3ROm|g_ab3$M$&q{I_DTnM zGNX-dBG%l*z~X9H#*+sQu?>vdX-Mee1b%<|7<99t!FSn!bt(!TaRy?NF9S4&K*32J z22_Viw*%%{>%tjV8C6w&l}Stbf|fEU6!<^6PVuGGbaZ?yeE|Fj&$mqNxo+&t*H{rJ zCK8@*j|T$VnKyn0q>X_8lE`ktZ`XPkDK*5Z!D#w`Wtu!Lj(VtCS4x0WHm{&$@VB4> z1Y#y;;tp4PjaJumKWz@bkW1nH;j#m<&*Q)#6AbfOpBvJLUkeh-tI&7>akytt_|N^_ zB)3NWcOd41Qz8Z|Q#u7Qv$O3VZ^tj}Izw`SN%0wIx6x+lK7E8GE>2Dx^H;6Nl1qDa z3+8`MQiXj&zzKa&vDOEOr|azSF#-{Jc-->j(j3lE>K_9Mf4~&jfHi7gV5_U*IXNVh z?CeP}+s3i0R27_Wdd=6YW`lylmx~LNg73C)Vfdf1=_0bn8h|Dn2iyhg`KB}=fyQ{( zL+x$d-Q7kbb+s#2y2>>et*0x$Flk#(((rj-VqM&CV<2~*$VrMYV#{dUHnw zG$anAt^vWr9qERvb=cLR_-p7J{{necRnGYSob>*>Wi3^V=S8}*+Nc44aZ*{z=*u|t zpwr8L|NLo`M-tC~*83=Z!s2V-OBkR9N)AX4fDjNhse*Iu$?0iUaq;)rS(5%ZsErcr zJu5H+jSY>iSv^nPL9|TOuht|yVqfyJZ{zmz@njqm7ARVS#-RhIE4$Mi*0ZNn?ST>0 zw0S(ZKX7yMNHe|M6?JsafFK0aVT~5UXNcFBhJSFM30CZRwe^zR7*ul|vh^$Jv@{+1n*I#MA|0>Al(7-0vd;xF@J zYH;#kwRWz0JS6fmv|#Jm>Nau!Zx^KZ9BDaLpfw{T7y=AxOeULAFHZM(5b#q`0sY>c zY8Y>@Ibe|lEzWA9#Q1l8?<}7gS#q5%H4ROJ8rz3d1!-erG;o4Jf+s~J-LK)1<{?v? zX_W)31Nld~A>-}ugp4=$jh=O~>Nwv3vXD3}NM-v}2h-*v_yEET+guDFGV;_983Vk= zk@m#7#Kr@YcE5dm{0C5K*&Y4cP#>jyW1yxASv_pJoEu-NOdwr;ST{5uL&p>o869LcQv+NHI8) zyc3uPK|r_zln8(TAQ~x{b0%q_q%JQT0?+fcp4Z)iV8x;Zh(Joc?^FKy%`M-ptOcy7 ztfa{pquJI}1xK^j*_t2by^lPKAG7Z7t34xMkXXFEyW1KwLttiR2G9L-G9;qQFjWh@ z`Z!=hBy!j-E!e>$AFw$)@dGabQ7Z37bA#;kR$who>cJU&J(h6wJdX~j>BzIn3!>^p z#mkD`y-^{32Q(_lbm6409!3=7&9;6Ilrmy}`*8^0F)qGe9fb83! zdPX0oYP?XoBXT(aTpu0#m>Ru`PCVf-nR)_LKr4m|rKd+uQHRyAapD@**lB;M@rU!n{xX!b^bC^jys1Fz+uEn(NHW%=kiE?niTTb3te;f%}Y>HY{20cTWXK zdF29|+pjGgZ2kwatk-z;_KM^&q8}lbC7MX7{unKD*T)YL!xiFYo}K3g&c8Ldt##DR z%m{%d4JB_e8<_b4QW9Omlt)Z6-HOnvr#nuon+6diLf+G@$#S%yYW!Mf8mL-&s|^%P zzz*<)sGw>Az%CtaRM&t0`~=j}<%k?1&r@p9w*X&wSUlI{RM4uUyjrfT9^WtfQ3|+P zFD(UHb{&@84o-#TtDvxeNR(zrpM6ZddGfVRgx`Kbo={n+x)n#P#=c6ziNl zvYIS~x?s&z7%`xxU7~a3E%axIif_&feFaAH(bmU(c`8AeaV9 zF+kT8OibVK^}PeS@dZ`QGF+Bd9q<@0(Rl`%eu(-7fO712dn@>WG1W?SCa8bZXTBNh zFlOg+C!R_NVz|Nf-$Y}D;H$P4uapg)AsWt(NG?k|>nfovN@$YE=GdH4c!>xgvT zE`ful-qJaN(n3rWjNPSavviP|-N1hFLPk@Q&uXHW`e+!E=Cht_<=Ib^?t4TDylgab z;jf#UdV?^<6yB~CR7m^ZLJqA$`ga`ZIXe*q*G$iOs{I$pV^d7TR)pMe=E407s` zB3GX~I_~-tg(B_bVZlW9;ptFEf`Ue0U|vJnFGO5SwrJj7-8Mw(o*jc zt=3B#e=H^@A)#-8v)QI`!5WBloK+YNC=F_6JX8+)U+|dI`=Ov0%|x#`kKyvSk^!Ya zHlE24SiLVqym9bC31fSEK>d=Dk*B1f_zd3UK~$J60~8vzCoMx z@w_nGPd$;53)oIW_CghkRsy*gW;!%%$JWW&2u{KGWC0wNkwKA> zRdg^tx8fx7JP*sVOMmnP<(ra@&g!hn+iH2xy#_yR_#efB3@gwR|2|(Oxq4@Gw{!uItT3XBNlpE>7V{WN*_H|9o33zjDFE9+ zo{tP`9c#d?KR%My2hvT$%3!kA?G^l}O8^|J;~kXs z@#L3uT52F=FzrNe{s$2^=47Mpv9iVm5Mlj} zr{8S)Ils=$ueQGRZ+Rvah$VNFd=X#*GAUqvnym&lCA&RE4sI7Y8IQ0@(eJ__ePBBR zZv%r|SR#H(Jn)aVP47HBJu@>7d(Gg@dHlX1 zS)b}=i04{Od(nEq2mllce7MZPg_uZ{JwXS6y}(q?1GlqQH3i1ukX<8^0i+a1a%sPz zbvcwpq|36o-n9n!n;?1?n>K%2L%x0TTmK%jX#CBWDDGD@Yfpzn;L@PK^|JI7RaFDt zJO{+be4vU!rNlKgVZ0CJM)+n|oSj{P4W5Sb^r?9_$Xi@31j3`=24S_nef?Vc-Bqtm z{6u^Vp<({T6(5RxFS-l`K8y|(e}lhnl+H?eQsRGT70ApjXYlrGh?L=eyh?ndnd#9 zUd!`2C>~lRabLfBvvVZ3P?VZ`0>VTIm@3%DF88MF$y3!HKA|X4$ut}i)$$Gw;vP#w zW#wi=3f$HjvJeOvsesG+g!Vo#cEdK(;Z)$o7JIhODgEoe=vqyPT3H28kFHP5CQ>qG z(y)vBo7>v(IW5II7F!h^*jZTw_eZV&&0`mrfK_{4V9a}5Fg;{r!LU*0hr?nqCAlctWjI4-#s_o565a%5;-wSEhyTa@;EcfnlrVE z?S>$zs)C@Fug>K4RZTnaHDIh|}H8DZAv}Gq8*>VPxU$rMh{`C^c z0tyHjoW~mZ*(IhaH3JuQxSY#V7RALK1MS&wHfOsclA&22rhgr*=YB7p%T8TTl2lg0 z0ZEL|al_1Viq>rjx`)LX5wq-~5}^wxWGS?S$@aVOlBj<9f@~R@uAHs!2d%9+YQnOa zAG*_^+qUelsA3|!$vo=2Q5yh>XZA$gCnFn}BmegWL8jx2 zP;Suc{E`E4e*e!T=8#tQD`FrS=!?MDw?qHuO9>mP|J?;6 z-hcHM;H+9DwQX+YS|z|n;53T=_otLekx8tMXl@HGEa(GR4!k7wKev+`wErFV#2OGs z0@mBhNTFj1zsIitp)OGXTm7GpK03*GXyL+E3?D?`8g#d?XPd^ z32m=*44CVB9cwn+;uBajFQfdw2T{}MTn2X=fKu}0)55*b>iTN6MB#W!Nvmv~igWI*`hov7g9-vbjdSzVkz3HomF?}2 ztoe5v`2QJVHEC}30DB17q)@1+^6CXcY0lOVIas2Op+q%R)t8v0t2I1&wuikVrZp2@ zjZ#ID;{^$Fd3n3p1$p$(0aYJO*O3El`4ct z*;CxfeW}~KpFiutz>27hOs|VbTox2~q;jplE($4iPD{`3*PEMt=-<5QFfHtohGV-P#ca^7t5@@a zpH=SoHc0-+pSx#4c$N<)jkzN8vO{gY9>k}p~Gz)OFAd{VBJtM*SdeM$+LFC{<&W*YrlWrLO{ zV*j&eLzV1U0X0-n_3O0GtIm0vJTNf|!mj^Zh&kS15+xMc21e*WvzOh zePJ^%%hJ+ykalTg zT&>gT46k0r42fvVS<4-m`Z7N(M73ba;|zKN28MDiMh(VwwP}7HYC}$lLGwGVzx|z_Qs?32WmM6_ z&g@`x)e66O48}KaufKQzq*i>!ld2F3CN9BfH9`saO8;%iLUTO?DkZzoQujg6Ic|1E zTf0%fX5#Kbs3VM!XR<+8l?&`U$jPvG1i7N!u+i&~hoA(Q7YpC^1GA;9<)lFQ0GW>B zi6X^Q^BUDwD&4b#O-G;{m^{9P9c>z2AYze{{sEJz8@ci6*~P`x%6B+mbtV3c#HoMJ zd_(kxLCfvWY!ulu|L%dcjx?51A~aY;LrSO^SjY%aEr&sAp!Y#xrZB}hD$)u0SqTF{41Bt9b2~m$*ySxRkAAb0k&d_TB;1VKAveTUdyW z+QQiZMnC>OPjL6R0;&yh$vKp3`=R@egsjTV<=`mc*jIa-${T^PDl=|Lsjk=e2k`q} zBJ=KnDCFK@DC8;9LutwgoT&l-WC3PDxK-cxgb&=ZS1WSJq4>T?Zvk8aA}aDd}r5<_uTL1B-=Rxjp2kKM$rhIzEWWN7Cf>KRx4w!1yty z@FjuBZF$Pw&4tRv&<`-zjD-y&P9E=@ut}b(5NG}>@rQ_?I<2zu1LW2L{q7pG;}a`z zGp1+et~-@+aJz9MHo8Vpt*?y9;`(CncE@W@j*cEgDlk|ae*0B$qEuJX(-V4fpeS^O zrd?}mxG@;oU9Eq&%>#t4hve4=n*&Jc`G2d;2hs{q49eY))Q!{syfXN!f!vASW?55V zI`AHhJTzJB18khgy+@k{KS>9LF5V`e{y8?SoXO&6Q7{Ws;W-fE36qqNU~~A5BPb{c z=A+b07`p}b6Qtt@N%pJyUR*3T=1J5WQZc(X8gDCjBqWet71FT;Q zSe}qVe(VDL@|(YZDlS>$Lmg!q_}%~Dd+Yz z&LghU{7VVsUMu0HUwLIu3&#Q!E{2dPYIoi}&Q7Yb-1pPC%{HH1A$&!ACi*CXmeKU) zfW@e8Ez8{buZ;^GTWY}a8o8wHjIHb475^5>#8>DtsYWQRP&&=4p8~dK&rs@oy7yb> zJ2eeEcd_5Y9f+sSVWGRaJSdXcYZ35WIwp(PMH`rF0owU5#_P$!vlW{s+)gewBBMO) zsn7*5G7WA5+1!z$JU)@ae0>wY_WOJm`?Pw-xy2U{^>eR$EjPN}kUwz6_xGM_oNpyF zVjWCg$`o0?%q{}dEDDM!*FF9Z7e75UP*Hn(dmm+-q3;ohK%RrpbMK#_T(3I4S?{ru zuyocC@xTuV=ED>LKtjTvY#`RuBz%}HJfD^Ue1158NFF2sGsbp2oO|rEURkl45{sz^&-#Zn0JhF)<;pY9_(`LWj z_!%u7Q`%l@v&7&CDM3Hvl(gadY_;wtn(51;ac`4!SmR-%0g(F5@dw9k{<* zLSNpz{Y8S&4VRd5Asr@c;;p|!alJiy&Mf_csN_okX`oWsnNY&rs~-8e-UtyrnjNLB z1x>T+;dv+0Di~`B^O!?i*yVg}w#rk63 z&~spjyY4M8US2z0pEg8maaqNBcZu({ny@FbCO7vC47{$I1ra+C0iV-z9J#3{X}W!7 z{?~;%I{YSUWb>V@*8AJ#NqYWhjPH=RTA>%1q`a72_sI$NXr8x)zB) zr2n36-_^P1FMJmIi1j95q+sWrrJgSb6eig>Q6{5v^&(N( z!wr+|jau*JQu(99lP3^!+vzX%nP5g)4%xJ63o{>EgGbPfz()UV$^ia(1h!_c3!}BZ zHHmJ{R~%HSi;GyE7u((b!Qh{}*vYG!0UoR$@|+~_9C~|keL?hziApLgQ~E*}xPf3= z`40o!2Q{>_vENm<@!YuMV>X}tClMN&PBgFG*GW89y#fdKm>?bJ+^%70xLxJ4Lv-pX zB;@5v!(2)Xx?BIfHLcP$q3_NBX1PWa5S++uW@i9m37yB3j?A^gQApx zhyqG?gNlHXBHbVFiJiT#t+RdS zUi(~VXquEJ{QNM_=xg0h(Fq{kiEI?rt1-A8Xcx4KMy= z^sWXq3UNmDRBjx*Y+f@zw&(m6JNbOiJq%?VKQvwGDvCGer&rU~fw+=ZN!^-g^))kn z)-%MyR~F2g{A;luaZh(w)sUxXuho09y(ZS(5W4pbd5D`q*`w$dfL{!M7f-t<`1%J= zlfx8~?mVL5`<2*=Mm-F&7CRqECQzIau~f5uV(S*Z9`MH#Us(Nzu#S#Lf{P`L>xr~; zr24{0*IgeR(ym-Nk+-w9siY#8?`lsLr7UYtxX&-Hp_E@;N|+3@M#3AkVJpzFy8l6> zrL8X|`%;Q#AokjF@kh$vQ7Gp`;0wFPQV5hf9U@6`uzR-0(~LQleIH4@d!@X?UNipF{Yquj}JB$~a%S*A#LTpS})- z#gv4HGhq)`xiecvg|&*iq2A0Mz=|sN3JU*aydjXO%)mc>Kqy-k@e4a4Df52(p=ZWz zk`bbhkvvzv{7g4))g3Dx=3azDT<-hXKofPDhLWdIM)RbIRxZ>z4yG&ESEFa{3EO;s z<}hwsZNgk-!+1&R_d<#=1aCB&#ZVPk)Tye0f`H_YVSgGfXDB>OHjQw`uN z1yKlmW2uV^OrbX%O@qnZOK;2|S7#F2r7r%QPTNJ-KLvFcUJWz4aGRc793`SIj})`G zIZri~@6deTJ?CwfbMyhSoAr^_kv@C$6G4mx9T6OrjRRy&Uj9w-RX0K6^cw4l&$sGR z{WqoIwf;xg7Ej~vdZPS{iqmb%@MKf}yp5S9VXb$UEEsQUdaYGjP0m`67NzSJ!(oab zQE()e4o-vap{UiNMCyQkryNc7Gh!4RZB%+P+|qMJ^OJYxmwJMiGlg((V`lb}uuR=7 zz1$U)u#dE_JJ>CFc2Rm%%(3lJe(!4hV0?j5uT6qyj$62AE)6IZ#{CCKd4zo}~ ziXd5u2T8FN$ulkuNZ^`woH_0VFmAHFVblkwtNmmHj7ig3?o=*u_?=Cqo;g(9Nk zF&z&W^-$6?TPJ|+c<@7ot=7@OT|w6y)*OyvrbiZ z^R3T{ilnY&VRWXkQEU9Ys79&9SkTx;^M-vGBrhPu1vMsjYy^zY!sK5p47?~RrI(j~ z=Q2}fe?P7DU78jPn?d$lB@CxRRl2v%;=Y78`fFoaZ7SQ)toFhZ)yXN{Y8P4>!$pSI z;GPm(3SN%h-L(Ol^@okoKxp#y^}UG}a=D@7Ayg;L5^>dSR9s}U!0sc&jaD^LUz$fw zDqznpFVE#19BvY4iQvy0|{Y zn|O`Rg9hyZ8FC#OSuUu=rC0&)?-Y=H)~r3~A%rINRAxLMvBYlJ{yf(y>tTM(GsQXb zcf&J;x0I!qh9@H!uN?35dy-D|+)%X*ou!(*8Ro!BmY48lG>@<3LIO@_Ny&o&8G}?| z$l6NESFW2oy)!?z=d_H9e<5{0&+1HPWhK*y^WX7m%I3jH4ab z%Ky~SllRc#yVUy)@}I;D)(>n&$l_-@gcRyJ-FPLj)bCbdU zLfAX;Qkx_2S}bu=?{enA`rSX|jfE~J0#WDH;g|W9{()=F*-Gsog=IWmPI$Gd$N$K% zZi(!@;03R}Z+v{!tdQuhqT*tDX6EMZp@Q+0eG%0Se^c%I-`u1?(zVE@|LtrY?#n2a+sxS$}R zx>~T*azfJHo*TOF{r!qDobR5;4(9mrY;R~UxDg;O^{jaX-BQh$`%T@(g(_OgU&aEzRr?vn zLgx^f9FEuD-CB!<=YI3E#A{3fD@PR zS=m2!JnKnv=M213Rr+DBr$-HGkL}5oP*6OcZ-z_ph9C+I!{>gWIpNQa351rYTB@O%AYW1kL?gVnde|UU&O}3 zlEue2?XA8eL-gX2KsjSK)VNxL%tMbOng0NoGN7QSXN;#;n+N!$e{055id9FaaW@c$g?!yPh zy33;T+=C^o)+mPyR+1n=fy346|9#%dwd{cro5OH*qk)Q@`&8J7nQ?tZxsOW17wS+uw3R55K;95MDUTRt4I;V?t zW52o7mOzz3Ii6HPhdHgL;yk5Lnq%J7viMYK!sH+$R7SqUWM6zd6pS^rDAW^Ol7Y37YrDQs2xly`q%LCcDt3T`Z;}c(3EFD+1 zm=`Qiv31Lk*>+B}me#Gg5e0`lq8j&OVhMJ(*h*dKWJ#!ceO-rPU;|BQ0T7~>GMaCU zw07EjI~R&KfAgfFqH>YlCO<;wN~tP74-@#C*FCMo@7lakgqZ#D_`92^+U`szkx@Fq z#5?+6mmuMa%k`}S@l?@X($^ERie-W4`$A9yt4E*OBJAVOco|+b_^_!mPL8{Diz5FS z4%(SaoumN4`c(O9MZV#?;0mZAg*C2ORlD}uXloBYU{P3XBt>1n+L}Jx%0$fCn`%u_!PWUHM}*IX2DV{QYVT_IO*DFw z16swhJ%gY4w^ZC-4AuuHyk%}#Xlt0Ysrb%j%Rl8__R-JprTQ=P&XRXlFe$b7U0`c{ zhB8VrDqrX|k@tW{0e!LSUP=#}PI}g*wiR(-2(IU< zc=6zTIwqz&^c$CBIYYts_-tU0GrgrXyNZIs@^PWjZS@SMGoG#%dDqV$ z{b&=``qQ~hrnQpjUC%aD;0`XbU9txup5@-u&?VHGM(yV&s^2q$plvC=Xp48P`gM)V zL@Q*a32J}349^ak#;sd~G$CN&-J`D&H{~2o9?%8qSJQ&nwucvHQw)Ns;v{# zA)mvOJU}YNpg@GYfk!*7Qjfywkm&}`iEgLVBN2UksQg?8-NfO#fQ5$<3bEd0)sqo7 zUa8!2ExcWh!#5dCPfZ*5pQrZ5+YS;BY`oo22}P{6>8u`aFi98U{v zX{Q7bed%@9TQ&IB^R3kGCkqnYI6T?e>FeIo~7S*%Wd}%&a7%@Gqo8WB9DL%JC ziB{p{H14!iLlR6Y9IthHqgdaUS2m9Ri}hIc->&om#AVmYK4}FJ?}gjIO83{W^v@W$ z8kg3k=e(cnJ*P6d6+EWx`5--6A7c#-y0$jF)57q8}FSDG+C)e}U)WSFGctgqO2Uv+>IExwIhktaYow5I$c?oh zX(5Rs4}U5T`Q4C4?eOkWos3%~199aTOS6oRU!-!(((~0PQv?d~J!X#h`SU%0$yQ@p zn|`;t1Mpk%($@7`zx2(2ABaX+E~xJ6G}=3Tg9ZiAP&`#o#@yqvo}Twz5h~_9qS%V| zhuo4Z4^J$Ij_E{hy|fz3REe+rK!EJX(P#g{?8*310$1WKQbcjd=C|h`Zm`%O?7tQ* zn2j@IR7KXvOx7w%l0(FJW^S%e>1wjgmw^y^L~YFjEr=jD z`&%s1@qSRY^dshOV)9Y^RIA#w^1*S6v!460-Pa!W3i=&mr4)|#9WS(aID2x2rE$3z4VLr)J{{q>FJ3q>AY9&F@TB^g z^mzY?6V0?F&RGY5ksUef*K=;$llr^0i=SNnx2T!h9QFZ(wx2j*T`5ImlJ5p@y?6aJ! zQ|2=-^vZR+-dy<1oEEZ_>FU+X!>a2ruJe0rYql_14$k4vcL--AH~t2W>$3owRh;Ex z9M7DIw6?VworUCaW>Rfr(L8>~Qy}Z|^_l0U(aewtFdm1?JXp3owj-BhW`C`}I7GE) z@xU;XefLdP!de)hQqjTWzQz~{`^(Q0krJ;v?-Iqoq=UW=B!LQV+Zz@bshv9Mh!*2y zw=1j~mJB~r!}S;TbJs0E)&T8^#O=F#OF{u)z0Z*UnVmE&t&hwWGfU5n3$L~^A;5;Sx?}giJgApLKowLMDlA?X<&zFxUOYqLpEs32XO)6t9Vx>bdtvWYoR@gfb3Rtyyw<} zKV3pIsI%EF;R#ILVwq;ep#yT|&Hh zXYUDe*8Obb9S$jbT!n2d=r5uT=4oirS}DoP6Q+ve8ZYJnm6_0=2rTX9bS1g6@7pBOdfPw#m)#)!U$O7msc53-$BPHLyb1co7TK|*A=D(11 zG_X)gHF=!vW%=`lCt{l(2@@vgj(;^LGbD=&J^L7zy6 z4)cFHhl5Ox0jb7<<=U9pb4O)GA#$m`x}}a0(Wh1XY$QQ(ts(-?>>M2*f)Jnbv%q}H zbdK*8e~m_!1+TnQ=+kSSvmm~tXWQppEh4)|OF5>_|5_(#-XIJ9A$>`Qy!~&YR0t3} z*Hi#F>YkLg_>N&h@+;(TgV6uMmjvqPerA%Eqs+?PRO$nr$iy6eqFJA2{J(if*?0cS zkRe6=zkL?||NUqnL45%CKOf=Op8dv?I(rp(LC1g7(<-%$nQSZKmP;z1m;`6S=<%kF zFveA6`@#$PaLF7w#oS6a4?A@V=BVQASZj;d?fB}Ins}T6nhs4TS?g=Q$UlZFr(fWG z2ExD1JQ5>E>e+QMd6*&RGgAUAMp#&w=nk>UvxdfnA44njkF%=les!8Qn6Rf*Dt#%~ zOqZ9bt1}AozwPZE@g{0)$;DF%MMOjb%$19ZIq07hwY{dX?u8o8sF)ZUp6KZ4z@$bY zQ^EUp?Nii2dk}~OUn8PrAzOX&jp3CdUfEdifVy9&X;XK1BIruNU6}3*+eWkg;lH~l z8`Msof9-toiNLxoGZVNg@&?6k3sti#`CyxMJ^R=2A@lCOGlkPh1<=!?!UH6=-I<+h zKK#D>+>aspU-GX!6HMa;1ez;N;K<_=5s%2yF5Q9UPXDI!4*5V7Z(}jPihRQC+1DN2 z@djw4D(wGu1&-=eu$?h6+IA{|?W`eWqAz(3Pj~qh$X~fDrjvz@tt!;?ba;^B|BT>ar|d_O;@tlNc-|FBqm+wuALs%ME`5ZKuGu% zhtib_Mpg-D^k$H6F;Fl?0D35Y{uGRJ1Ec%j*AMYO5-cjsmM{R~{q#>xf3{jx$0_`&!RbW-yDD<95F zOdi9VCR-)OqLA>-s?o|h`-8NDQ>g-x_vo*+?Y|6GJMG==C#&sLB!b7nkdmNKQzxHq zU%Yicv%Aheu_7-6q&B^kZ(N*~t~JJLTa!E>r$lDd^&Nvg!nW^!QP#~mlx`!%3ps;A zP^Z_t_dr_Oa8Mv#79exLh?AX)Bg@B`YtP)NQ6{Pmo)-+clv<3JKL#xiCJk;h;2<=_ zT2*()JB)iay+tJgd>+P(g&*?sALf<<5kL#j7$}!e@06NPfXHa78Uj)O#RnG@HfsRv ze>*V6@I)|QxW;PU^GGE90N290LKFSE_xAFa8RUzRs#TzJ!@+S4de69dqiE9$hRKHQ zgD8L}CuQUV6%O_1hZU=j&I!iK|Hgf4uT1Uiha@Q=$R57y!b}qU1MBbd&dwpK?oj@* zO&MQ;`@N5uF~3kiW=|z|5cEILZej=0YY{62W@ggEe?ZygrJQ3p>qVTUQASD5lRwv0 ze~siriwy94&|ojXAd5}(Q>>5e3-b-3lM4|lKi>XQ8!5HBjRFIX2ls}uD9J&4@{e*o z!Nah&4(Jv8DCaD6^5`r?YPeW0O{1<)#7di+)2RP44F>4D>8}Xt@;w!@bw9t%PffG9 zPunB&3RSDwqNN*Ov;eB5dO6f*bMy}}-nv#=oCap5Xe;#^+H1GIFPk_YE?bWrF@U|# zY34WG)<70n{rT2HW2UX%8Xm;dS=u;zK#HS}ziOWMg5})LsKdK#vDGLr+yGD`ZdjXi zXBg@?)N7U)M39ZhfiC5Y9wDdYI69P=QY>*MGy?pizwZ1;t%O_IC@$_`< zYQdN4E@-Qc_EFMm@FZtjuJ%`S6(`aL$EIXkD(7h`&p0iQEzy4d{JE|5rSVE)-L)@K zQ4M_}9uo4LC1zuUAE9}5Ey*yD9NG$i1>nO~u`|~K!}w6bzIy+j9_~yU3B0hdFs&J3 z?&W!R#nGh8lfr?eC6MCO8_eC^!fd}ye%rIn zq}q!K_`=qCS>Lkkjy5M6CLAV0Ytpyx zxrv4 zkGWWHoxW5TEA^7{Pc>(U(_2)4 znDtfK2|&-eV=6~wdp@kY2Sm9+6fX$afli?YZN$h(?FoH5VALJvE(lLf3UYs-hhfvj z;hlgj(fm7A4s_5};fGPt8MVO*1v>Yu3!c;cmG-|%IobKv@alY3rCT`G6>-(OtBOAO zR3M~TmYV%!ZKz;o;GIIZWRPlklkU0`dMO}VEtZe5Fnf~H58$|=U;(3Tg z!UBi_Q9}(@SSV9|5cvtv5%|jUYF5P~?<+KUu)8 zo?F$#AFu0lODHkmr2+rK54fRUVAQq6SHx> zZ-dq`5X}5vRkep}ly74vcwF8&Jy;7(p$FO2kzz3CAgie0^YWEoJ&nec_PAmKtr2LX z*1Db9F2BZtO>)vQ2hwA2w`zqx_^-*nd`S<#o~u^88)tZz^gFH2Lu1I+|{8%9hDU5f?0ls)aXjovvC+kvYKZ z?DrsXGA_#(uXXPSq)_wlLdI{;W|11Cb`T5zUH7RLnPVh#UUsxclh)A*AB7XE&mQI~8D0N0g!q_AGCWki4uXhtMurnnkCH?&U z>)#fB2@4Z-brpQHFuoO^4F%8p^#l0ChhvK)$ztE!++1va z9pt{YRJx0U`ktSnn&(1>lWZZb=7B5#KiwEqwGGS0wC}qo@Y$2Dp2jg_+^8Xhgw@)b z4G1>(d3Z9UD~{vGfV=mI^*#Ye4~I}_2(7`pJ1qIU<;x_A(1bX8#{K@I9#Q*be_)924iC4&9qQ?OdpP6>;k<30sJLx#~CMJe6mz;qro*) zcX=8Zdxnql+k_RgQ0@Ukkhc4z>+fHr7QW#DD6qM8L8`1dk!`>oUDUsaN?BTz;a?<0Sd)=Y> zYo(n)U|$^y1zckfAveWlzkV~Id(Y@C;sZ!RpGZmRWN$u*6E?3``G@9EDqom? z5pp>|rtwKhx}#-Vul3jw9x$AEVlc>PS)R-_Lr4fXlPRx{v=I_K&d1uq#lXGSno}T~YPZG>ZWdpoVjTv#6T1`T>R^uR z1OLic8P#1YD!KbFG6vZ7n{j}EA{|O81X60Vr$1z11cZ*tbsKf^Zo5+=4;%v(p8qa& z?!y1TNEt07V^EC?i*VWC=Ym?WEXyx3wY#>mvSQq+e2em~xa)a{jf)S_{8GdU>29Bg z_B{OgbCu2qf6b3}I-+q%dJ>0+#pf5}-KTXb4cMUD@?pLIB-ftHyyqhbY3wb=TJ7$7 z=Xv8&U#4|$-M$T;e4Km2 zi5`_9!z86vW8&b+!sj~Q>XcYz`>yAH9HfX7J*7^sL18_1R<9q zqfRn-2em4DCLOnfJ7JbQb*C>L@9qxm>FKeh44asN z&{`(F6?m5^1{tf`(VwHl$r@MKl{4GKjBjthhXTTJzgGfW&pstA@WElpo6$j0_* zqU!b&dOM@DcFL$;lg9S;^U#GSpp@W8*V29^f?NR5XVSU;2=Z@Dv;Pc(T7SyrCbwPc z)Sp~R1cRWabvfgmzYJG~kbH)RhfmvAb+7tcKgOjL{O;#>^M`cwo6XY}A0chMk>g`W zv}@N^dXhbjzrPYg(y;axJ7ENfYqy(itm7mgG;|5#m)_s-rFL3Zq0rwQbzj57UjY6I zJ>{i}C0grE#UtcRXQ=WNP3fPLkONj~TA(U%8=NprevzGDoP#A(Di~sdL=IHtBeiZA z*KX{CERHrX;|R%bH_X2Z19yPz{X-Jm8@*( zmTj{X@9;)dwG|HAx36EjnuEznI8CP2I7XFxw*B5iS=qFd*qz#Fd9{Rg6^dAxEo|}m znvl?yQEX(rzf6cclQl<^wu3>=kmI&(&w%%eh6;3M#ev-HwKFkAL~|Hxe$$PB@Wl|;K}&;-W2VVPMqxYbu@(7~cIX(_fu2jx$Y{`=AY@=* z6gk4oOYir6n1oULE&$%c6=uXtF+9LM2&Rz;MI*RPA!A_husxE6MS&Xv1^lr^Z=3$I zqlWw#(vj1BeV;)&cb_qb^o|%Pv#Zv+@|C8fc(`d!-o@^W;cDpaj*1)Cyk$?NqrU*9fb`-Eq=BGSB)HUMz}c5BMGV4ej>;OY)iualB9_N$@tCx@48}EK5JMa) zw(E3Q!$nSkOPZ?({AAoFFJHdA3EFTVF9WlLgT-LzI>bBZqT+bYPUl-4%txLc5I z=>w6g7$Q6f0vP<7vC%r_@JPQQ3hLrkoZ5suWbof_%p3Hge6LeCj=)L1OJyZHPYglq z%G#yoS?Hv0H4f96F`f(NBe7ECo2gJ_AU>t^PG1u9NI%!S8LS+LSmE9J2*Z1R*ctSa z(JrUh8`N*I!!7J93IA}XZ0$LH38-Jj{~0T>axoEre?>&T{0wy6n(g7aw!%Nl1lJQYuEjznzOkWSSp2A2;n{9)HJ5sj=1>n#^EXUORjtaAQr z%X5ewgs)keKrQb1HTLvCA_tLNIl-v0LkATzv8LR+B1Q6$K*wQ%p&!6_CLkh_eq)1n z{W|+yO-)L$EX@+V2bnAiGy!kbnYGkqOCkT_f36KD%~D-I@tj8ws&B-ESt+}@xn3!; zrg5reqd)(K{IV;~jRYs6v;dSNSBK1A1}^eAqG?_kO#iMT+1;Ka32v4&YdLwti3Q*Q zI(nQ?Q2X0Lzz)26O)A$nZmEcrjdi9Z)@78Z9qSCqkJ8M~`aZ#;ZX5sw1WNK#BIMH? z9;O8$W&$#@W=N=5H|nf_7_2!U1JqXQqv)Vq8RJVt+$m=dkTZ=WJ6fJa29WaHfis){_?+tsGRp7b68L!a}ZD06f zR17|gf#B~AKre`p&Yxt>@x?edDr_pslGN0rWFsAQfXEE(DVRd*G5z)Q9Sd%mMw2NM zI(W1J3jeHN2#}mYb!=AF1=3#ioNLtqw!QCmBFSnB79xf@P$MOm}P=OZyAML_Q9($_QZLH z7*w@DoSD};1?4id<}4(q$d!us(y7ac;pS)F_xO#UZibA<#Aq<&b^Pt+;8K8^WuSfjcd(5>5-dA zcYT9XzU{=2Ow$rBeyDmoyXQYi32yto^(X+FEpDl_A^FXBvmak$> zViCe;n$k7S2Y@M28yXtL#cP>N<~{;JSK@o%l-Xs=BAo$x$sbQIaUG2>eS^rHFhHx9 zmY+Y{)G`Kco+kuUn>LSyMXYS#_45AwR%?6rqw{Ul(}S8j zJSsd%vl<~Q2&2Boyf`2$2+!1?|5p7O_SBF0anX@4V?MS^8%NVK@_{gklafjU3534liHUE+rk(k~eP)^?9_^AF%sBmmOEna3}t?6oRzKyr`)YV^K znu_W4Gj8nQ?&Y5{RLALBj1@3~1oztTh-kiOg6_`_OJ5@*U=lvL`glCYi%R&V+tq1# zmCN>iF`^ZG(pN~k5k+EVfV4k;{7R`P07H=oErHwaMm>e7wXJP~IkzqC4$+neYhqd& z2TBR{>hJkip|d*Opynw7Duq&!%tj4AexY=@8&Y=0{(AU;J?=O6v-86uvu|UY{(UV<}+bd*t7#!Na;=`jsNT>_ zY)z3}`i@=a_B)^qw|hrQwaAF0+>GNHUn?M5TkE%Qn9fr?%y_u7 zlYtGwV?3WMK9?JhyG4xRZ<0KBcNg~kga2Ax{ZH#m8KPE;b%$>5cpl+>PlQkkwu6#Q z_vzuL+?Mu*KcBM~u}(tU=s_JK+mQT+v@3e%cjdjtIS$bO*Kg`>KVBT`hG;IGu-|$e z@g9=u7Y6-$;2;hYvmbXHL_UJYNidIb@%vs^X<6C?$Mbh&T&9|j^oa-LPqzW*x~xQwy}g4Hvg z<90I+lAz0?Tu(Z$o}So_QDu2Wy^&{BLZen$D}B}g!tfzs zZT>x9nEUhA+E);m590zd5)xtk1Z+9EW6Fp-n!sLgxd-(w(|J%&mzHYL2R2bFa)0sO z?OwXALNNA0fdJsR(lo>}vP5NTaQC;_7Gi_Oxb0 z1%Czxg7dX3Pg;%&!~&I3%%>{6Gcz0R3EUM`SAPOlG9SSLQ=u%MMaONAR3uMzwRPLk zeo+dCg8f_5l>3Z{S$)->AOEdSpuUG|?yFru!_DmjN^{ph(jgMWArzEpMV5G;R82Im z+A&Zdet)cU6-rmlCV2i7aD{^EQ^A7HOZ=m8J1Twg;71$`o>WmD-C&~w31E0c1mPWu zdgQ+3{z*|~Z)Lpvys~m;aWy1Lg#wL?i&pR) z1zSY(V|Ml(HR>%l$trt_6TOw$z>os1*BE46IMYW;b+2Em{&o;?6?>WS+6Rxcp6*IZ zaYYs10e_Q>2Q_dYpiMVT-^^5V49`S5(~fZpj+c> z(;LeQ^F&`7yl*ive_(t*!|g2OPJ1AwzwX4TQj}l@4CDaZk;}~xD#G1RsFi_0Ds6>@ z73*s_JIACN`C4g-0L9>OR-1vbO73`awrND(>tfRbD=%+*{nb=C8>zB{#wH3n!P)!Z z8-#v8O@RajPW`={BVIlwSB-%bwRTAh!zBa0+7mfRe&~_hb34QN-CaQ05<>NecJb!g zU_qtjlf}Tg7W3NO*{mYhe1hlakJ(l@VGK*|WdU*VGU z2{t2TGsgjUN!l4j!Chc3*nnzkL9>Ppgnz-U15CP1O2`BSO_PV#UP%ki z9mmHdSVx}GhKSa@=p+ahZ}@oW`Ivwv-Z;#WoDy;24sM}LY@{fg5(DMacLVHSto64HTe>V5 z9glD&p7rmod~k;9IW|Wly%1?17pCCT`+i5sm~*N2Q(ZPc`x3L5{FD5xbF;J5kWQ*G zS=vT6i&M{W+oOvQcq`ABzOd4c3;086`mh=GtG*2p--F|v-)7S=d-{MBE|?vtq?VL;feA*T!xS^5 z0kG!m+k@0NlBQIFKL_tUAOtyZi0L~|!*<5wAa|%%yn=W5AC!BBH9Sx=s zDATsqm~EzMdgD!yQgpWE^5tijmRh-qFdS3VJI?}uINDb*^>p>*R3K#Hr!`Zada2&< z7kQYQXpGmO*pfWo8G*wl64hj9Z(}svBR{{eRAhOvirw{&d>C(wA1M;@HNkqYaKeC- z5Olb_c5XXe1V>$U8G_OkFtrZ+j-uijxMo*!K!5^K;874&a2c)<7Q>wZwN z*9R>9aD-eOQij$@Ih+}B$!~xSXYiLVKcx~Bw6&XKelf@^MYK@5f9B2`>nnL@3o5ZM zVh1f?#`He*%tM%HFmHXC5YHEwAapvtIvW7el}bwMhXIUVz|ixxvXRl>`IeiT!_kmU zFZT$4AJTD&trV_}x+mIjitDuNZ%o2-sO?(4_BkLGuT6((gU-_Y4WklBGZ7J5`vK6% zap7-_iM=8IG_|?Jq&J}6(lhd9`x}Y`Q0olIgLwYOiSFHj10;-YYhO#<5J)&&}h( zgPdf{#<;Cd zKcJ!H44sR^`-1D7ll|ss^SQ&MZr8Y4hsy`@O(XFF1Q0M1-FCow$7a{yUSQo>K-+Dq#Ratj{X$iU5NT<&VLD$t z+AfbMhTD5nB}jTa?w5Vo!cZ?|AxN2q&#q0q9H3X1 zEJ!I|NrxpEzj|9S9{4rrOSbh`k{*x^W9`V5!{Xx?h9@C={Vd?~e)R*I%i$(^5>P-s zU~L9PMe#I=dA|^o_!1tz%-f>NdTX=-`v3@sot@8&d-jGK5NgHtkEq-=-z^U8VRs@lH5g|$NOT#Fn2%zx$2FEu&^EG#`qn&u-j~f5IuBMpeGe>v5(e_b8L$0kFu%=4hP26QE;D@OF?b~g>d)7T)g)r|R)R#Kl ze0^}y%02pqC3Eb9Nw{ysYjv!}q^>a3)=5kEZr zl-%_{Xv0h~We?I6=&g6??lH{AcTd#`19q^`5e)FeAE|$QnIh=0V58f&n4~ zwj4UN^qCX|)rV*^MK9uxi7S-rzXZI^x;l4*&|+y%8+KbJmUHgc$d%p|RM#hiaWRzN zpt4_Uox=6$U*;sm^wLa>i;KItLey;G=^GXi?+wZQ=r5s>8e?+g|0HuL>0b z)+K|wF2h(*4ni$(ta3F^Y-R9)3g1}wA{bp%Q7M9qdZ-XwKN8#*t&twONS@)?`aP_v z?!yg@pE+-q75v%@Uj`GvJF&xwe6Yz2R%^NHd*s&EDzyPf$yjSXE_NAl_U8aD2WH37 z(kSTv+lt84=IKCwQ^P$}z?`+R__YErph_+Q5bl!hj1 z&)~WJyX?EOP89#zwSWI*Lh*l^=EnTb?u@0v|5O#2J;lcOhcUxNH%W^k>-YbBWciQ( z*~a1gpUmvpH1hxJ%WT;BQ9x?NDS!VCEaaxHt^k;|tTMK=xF`lOWFU8=eYD63>2F@h zvRAQpALs^R<3H_S?sFhX&?62GG$kdISr&liFqQ=)Yn<(awZE!R=>EITq)a2tNs}(+ z`*)Q677{qq2EV@xdx{?pJ<1?j*GtxDsgd`kb$l{egZGGu8?0FkEzcsWUTx%?Uz)YL z!8{+_>JsmvoU1TT9fOq3f3ZtF-+s#sZ1VyLISjh9%xAzrcVjGm!trb}CFOB&Y{-64 z?%tMhewR3vSP%y`+M&FSal%KCc#FgC9+pHYX4`W zh^Xi*MGeKAVQ^Lh%^f9Fjtxl-#vAH?zth`90{KMbzCbFyGb9$Q7YnMyk?rBZyCw`BD2Q3@ z%fa@utE)0FN_YBln_c`ZIb<>sTCu{wxr4)ZXD>$_=Gtky9MH4|wSr^7#yC3bX1DWB zu7Vd7ak=^9W zct@Rr(;pqa z34TQT#y?lzq%jF5I5jeZ#j=_amv1REHKAxql4JXDYiZ+>B0uTLw>9P!8=ljBf}i!< z7sULy(lEOM%-*SJR@gkyF>Q(KbeA~#`g9G{%(g04I^Ye!L0qYXwJ zsBUlaf$jzcaBP2ko@f{p#bx|u`3@R9AFwro*_YuAXW`A$t`a{g_6a9wko(a)Qz=hh z@amaNYu&+z+lMO{-PaD|WZq1#I2;#eeC*X!SF7;}FO&MNQ+1EMude%#R2C`L-0Fi@ z6c^h5QlYR`jZ?-p-XXN#po-QT7bIrdJ-d>+EAVHcRuC@6*m1yMc#|aXgM_0aH$l|O zK;HE4dl+&XPu(A9-p9iWj);(~aYC$*)x@QC)EdrSU@_~QKZY4G?U27bx0As|0X|Zy z7QcII7$F!Q2*01KjiM%X_ne)54%CL9l>sIUs;`yg!ft=!nw9lUYTHjD`dqEd8Avo! zBCm8%NtS%~QrRu%w6}O~az34mg3^v+$sKiplnZl7meb$BoTK1?|+ zEle#f+oz^`7Do?ZPH#ifb4WVh9b3Qj_ba*$Ch0|^8Y{#~N(E!t<(1_;&@w_x?o~93 z?z%K&fyRI&Y><*Qcf&1&=JCl81y?z1Ufn6xrK1TZHO}mf&ASc3%P+(&uKhGF**(D{ z+gr;MoK?vaxH!X{Z(h>U?nsq*At?&)jMS_Wh& zVsdF+fueTsZL?Eucu=_;!5uLW7#&{XJ*$l$$u7;(+Ie{i&aHF zNRoX79X~-y#rx^eWYyu%)+z=#a3ROYwpyJQ>o1k)rp6h#YfU;^E>w&0;b(zjA)<;5e7aulM$r+0JcCT?8=m&`4IS zmY#ros~xA5`(g&z1}$#c|}?*d3?%r1gGc)k~`I5I(h| z_+QaSX9{Xxm49efmBh~RUi!(-G3do{f9PKRg?PYfviy`b6gk>LgJNa|9ZuwLS8u^3 z@?EU60p$;w9JpLw46l!t;rI<+{TbDy&>3}xZ8}~G!-xyb8lNU5Co}M*vrtp}sWSFF zrYj}_cuqvjwI0U6KB1?#BrbC5dMPg-nz1(5G2`vAKB`_|V+H_VURRtbY!(Mcr+e>tM!FLT^vygUrUjZL-9Yi;AJ!K><$UnxfuYJAczyU^BhGD4RcsZq1S%|Oa;ujGl$5yL z$ity7%*zN`M`GG18zE;M9UY^UrmS{nU-tT;gFcdP*>!z3~Q1JvOTx zTOXS>}nJi7uSVt#8jN5jPg9Kl?&rJTG>mQW z|NcgIRMF(6wo}d(yW!UPG_hpX@~?I-IA&S_v?PXQs!uv!K74y%@~srlfb#d_d6{#y zL!H)HAHCQyzxw&^-%-w)dn}p$GFu{LmSpl_uAXtzGHKadF@I+A`XOvS6MaSP)aW8Z zrsf%Ak3)z)EzxsUXV4S!<1v}0v#4uW#QYTv&X0Fdq@}T*L7HEJTy#c;3c;Dne;3m? z{dfU*ZZRCLX~DQNR89I?)4Lm=l%hQ?)SPftW16i%b>Q4= z7%kr*9^awy74wR*S3r3`ge_HU7xeqf5Q4*io=Wu#c;0F!#I5_OkX6|E$*|hWY$?B0vD-yZK?~qCiL$R46}QW zv)F(^x9FlcD6?J~1p6@IQ+6oIeS-ulgFWZ_DG4YKx8Bob;qE7DWg+lm`DxlEsSb_} zP<*XJUtjhzt(L5MO&UC+aHFoJcajTDQ? z)RsG;y;6>rbUK&uwdzKB(C=yI!W*Slo6{~6n6W8&Hhy3GQY=%fS6*$mXZVC2-~XKr z;Ry=E=W-p5m9vBxFZ6{4c}Zc#J8?2zGU<))bEB|CW>+I)rC!qyN#f$-4r3*^e@?9? zuA0>97lx!?P8!lHvl8tCL)#w&|JY8*9QOZD9jf_v5hL&Y4mC`5F)m@}M-s{8GH-E= zm#7bE{kMeG5LZ-cHYgh0XM$p0Tl#de@hfVevwv(!UMt$e=}5-yPZQ|!BC7+SipbK=YO%&% z;5+wcd<}?QGG;O*2iy+7VU(W+v~wQ+`t=Lnz2~CSVU~b2iYK_Qug{pM z_2l$`qzzBoRgJ<``wsFqP7FL|6&bkJj!{wY!k{fcm_;}@fXCLw!D>v3%0J=7vt+6! z;(!LLsjY4p)XqHTvsCWdsR#|zdSt5OV zu<#H;Bf7l+iKG55GSrTDbi##6x-D;Ms1xOMzVl>so`%+~v4_ z3E2m$Fw`>G&p@^YL+v=ez~3)?r%^DPeEL;*=8W-lYI99JNSw^=Q%~J_*n}kClQW0m zg$hG$o%Oh*S1`pJdU0IXBr9zgK&_dZK~HDa?@MNT_xtO1BRWh$60CDVl2`gz`=a<( zbQ!HdlO5UX z*Gtk(A~s!44h%tA066USUY59pg&_17eH8}AVKN6+Om)^8_(=)sn!#)M_Op7j@RkTC;+hn zEFxe=gtkbp^gE-_x@jT%RC4!`G!W?A6TmHdr8GMmq_V<)Tm{Xa|GZ;Cul7jC@roURe-9YrXK<>pC6wvLI6N@zRYe8JgpvP*729yua07cI!!Z zcQ3_y$6kB)EK?hfD`@7i8Xm?#8xfd~ z3S@EhAf*ISR~AZJclT_tSNxwdvf4+7uQ5QK2=xdQ!%jluk`Znljaag}qq|#I-hq)> zURTuhpCr1Qv&hA`Qn-Zz!0Y_f4$jF=I%O=ErT=wmg!POq%`Lcf&ze0{jf~o_6KEpa z^FRQzo^4{b+nV|aPQ~xoG)Dtak+En}OCIjV3iKC66=5>hia0_L%(yoX~4zWIJ$kj^FgUKm&WEz?$~@m+P>Jxs0u^l^V6~%!!lO zD)I5xZ0{W`&B@|oTkqchpM=LSf>*7CkA`0QM~L&bc$!hF^P95dC%4qPzBppRVETrx z!MF%RK}i^XMHqk_*#lg|nPfX1BK!KVCnhHu z#}C^TEF=A?K&s=ToRtdaK+3#tDJa1FLPK%r)fHWIH`dpIheO0OnM$TTF)oi{&SKDut9T1gK! zAd=q&83Fp!tAZdM|k`bQ4<|<6a2cMOABcp z+@EJC{%E?XQsJwXJLpCa4nC~`sSw#E0&XA4c;})#;OBin{X~8sN2o#wQ$*CxL5{ew zF$-|-8e&zZ$r%t_kQ?re|9TLZ7?Jb@B0(@D6-}0IFmo$Q`SKS|P|sr^U;%M)Zb{qC zMo&udizGS@Lp3@IP*xuCQ*<$-0TWAP{*l>+AbyRO_R6^+1QBWAb)nd8PIZ5{;t|DZ ze??q38BQ42RA<8988@MW^9?b5ad_5Kz1yb&_ok;ex%d5OY4DcY@v6DDwbFkEqyjxR zvB4jE!i7QV%Ox#Hdm=6MMWtXms^QS>JY){~eykW1I#Z+xr;5$peqpzY$4$e+?rY19 z!(8=PDQ^%8x;Z!w%~&)&Y88l+kzLHZdHfk`K z4xqIk$ZE<#gp4&Y9C^tm5$gC{f1teP^T*SUR@Ilp9pA9?mGosMyu?@}2%uWGE0Nbs z5M(S%Ds4GoBkZDVc)8Tgy+s z%SuSR9ot7oGGhQ;t3LeY{`z#z9_BCanrr@F52&Qb?mN42`6{)8Qfl=vX!gd+Zd1Xc zl5{^OebqBKobeSnwa}7+4&Lm$_iQNi>cs}YGp{arv#C#(pExUIHhw5ARekvgHft2? z5oK%b=g$teZr?gKYL;*1$WcBE@{Jna*)^C&#o8Gb1dJs&!cYS%I2>f4BRk0(#h98aJ^|U<%;#-!_q@ zw{eMlotsBTDG7=7=fe#i#D!lX`S>gruHTn)7+hS$N*l0r&hvN>3|k01>OTNi{$$BF zgd=16E5(v;M0{R#bu5fa=JB$5B!IZ~O0GW{$mM|~ysKQ$K;=P@-u-UiYE~G7YlY2jpYA{Qu z0VNg2cISs(p2S-qOh(Hsaoz^0=yglQAKw7xS%L17GLi}eeapVGBb1?RO`jlpO|{Z@ zT}fEeIRi`9J=LA7>~%=GFhr70XslK^Ph8_LS9Kr#V_^IB-`ry4w%oWbz7$HZV4L^E~Z zDTWF;HD`>I8v@A~o$k|JLnGE7_2D3>#&;Sd zWiJ-+!AGeBo?*Gk#5HKf^q)+Gd1VEMM>iyX6n^yq@~CXJi`%+e<>+t_LJM=_6VIzx zuYRgbyVwnyUTUVxP~RnZP>*^f*aQ{tcFr1*#kx+`-N`g9-m6AMeamuN%(XlM!HWoZ zc$8t{ch+81k!0ng_!GPFLfQlz-_7z1hSH4@h%?kcRfqM>;<%d(CC==eW&|+Uc$MW> zfcVCp*ONJ5=+vUB#%A7iaU}g{wN1xm?Q~{!aq;zoADMJ#JL+2T>=7`P8_ELY9pSYz zrQqdV`=mRZqb*&=!2i^j*lVj5C{V!Pb8tpztQ(8El0XZ7v(EmOT<<+lQTU48Au3vK zv?m^BJLsi8fgSqtrC$X=7uccQZ%SU@ytDBV&Oy~B{o7YSfAD7poeasleZm^EGL(wm z+Z!B~z)&mGq^ntSt^+G3dFuKmlTNqXYfp?-><8*&jTv-O!(y|o@rDAGoETB91 z;QQkm;5{NF+)_4DVae0L-2;&azQdZS&DL0G*uZ0fQ@uyHlH9<`LY8b$&u1!T?B``< z{+ha|GMZ+HbaO?YU7v3{w_mzPq<-(Sk3U9wZXOw`dCi?y$BR`xMj(3w&_h`6z8Q%H z(2-D{1*toQMmmfIKbV6|T&Dgzl4S#5P-}aE&%cW~rG9*4))B zBQMZDK!ydIE?8~`gM{xBAU7a9K>^;r)sypwhW-6WN+i&NFcUbtDCx416BR{Bd{7=T zuVA2}Pk1KvT5!$Yh5IGJWg9XhEZ$$BdxKiGrATjd*7%|Zj9{vc))OtC;hUAXO;K#b8EadCQD4IPWz)2#n##_Bc?y_jrmAq2ZCPQUQ4q$7xm~&W$P8cQ z6tT28Bjo7VmA32Sj`bJQ&FUjJo*iT6j>=cd#;&fsf-g-)x3GX+j{RJtS?dNHv2bGk{aen2I$q#;{hu-7 z|8M@1@?Wlla(Wipr))C7(f=_C@9ms9bUbk-nfw zgusk{&k>{RBU3cGeuD~tuGzJ<*7i`#K+~J)bG=0V1DOksLQJ42Wj0%Pfb1f*Ivc~I z{wFZZLD?Fz#xm!PtU=!*KUQA5_Y3I> z5nx=3f`mVWJjCcg+6Md#5P+5QwgS_@RrJyc6*Bw=03LzV^&etZ^4}n@m<-iA(&q1c z@(AZ1fy|{FD)RVCe2B%#iG}iu&fSgZ9Rs~+@)Ho%-X5#8{b_2m*5~oX5BF?{>Yn`)#}@t7|>p{G`Hw%|Z?+ z1XO3?h999Q0KZW}(E*A{lwu>27jKfefaT>{tTO8bp^sN4av z6$9E0Kqim4xqaMR(UM|$1z2{G(uDSo4*ljHNlRPnWT6YOk@t2mU~_6}3JMIIdyd|m z)A7Pir>cqMYkB}36Y~^L2WRG7RSUqhx3>qYlGi1tDgq~4=z%nU{gCcHpLhb;G?1a> z0CWeC6~u4R63TW|NS0`@SDGat`M$n>D9!iUP+qa!%KUZRmK)p)7;?>0Qqs7LKR{Ut z9}03}yW;O-O5)E_vpc2abRL1%Ppr4!!kjE#Uy-=^b6$H&PR0m`AR!^q7|)>Yx3smT z1{5>7Cs{mbd*evEk20GVBsNr#9L1EGJ*1&oNgB@VefOIPBbKbCAj^);w`sR+5KCO;@n&TWC@R@D$&s zVp(NM?|BR-+P&bJ`Nf64Qd4KZP_Qz%xVd=)?X7Kvl{O-~zNuIjj4Q!`=uqy=ReB*e zQCwbrTByZ5z$Dc=PAovELc1SY6SDF+hfDu5w`{W_ab-y+N8Z<92%qB~H z;3|xKZXaz=NKAxveMx4f??yG&x%)NK^Hc6qsTpZwk76 zRj)ikVj4gtBH{h~H6}vR&Q3YijxO9PkVNq1>(`t^xtDP;p8foS=r?E>YR^{{5RoSR zg^rT`ViX3}G#956zDe)6t*Ji#{Ao8KU_eafw$Je62hBPyD=R2-K+|QRfg|~5?V-S_ zjZSJq25^>j07|N6zqAE%^k|_e4s?pd&C8E>gLd;ThvyL!_#o)w$dcB|Bl@76-e^!H z@lyx@679dWvtlw76~Ae?{PE&rq-Ed$tUd4!2$&h!Nrv=hcB@veW#7wJ{g?*vZO`HTa}cfSyg5Yi?g*X$<}pkXC9{~G4bx*-;uU! zDnKpmDbQusGt}Q12<-_4hz&3!FeZP*!xPf|NVBb@gM$C!eX6;NZXoG***pXv|J-#L z4VVF$HBegRAVtnOdIi5YpcQhvH0fv@5**v!3&7cdWXg?AqS{4qCV05XQ=5B=m^)t6 z=m_8(dmU3*HH+$dpM*tk#FN$2fBI{^g(n6+MuZNk=UU?L@XD$xO4B~JO}m40#ES|x z$(I{!&!>7?Jt+{dSZ~a1jZuUhU{n16LXr|a~usWn(WghdjiIvWu2r=&8>3>PVL=OVoFMD2+h7M zL$X`9_S9{NcPw{oRziFlXHxQ?0KuDt-fn$SL|nAL$j}_7qYY)wjo?vG^7j`L*sapN z`>@p|JFeJB<&y zD_DQf7y)W_>(*_!!INuC#)^t;QJPiva2L}mD?{L^&+V{XqsE#zd!k-yYS13?MMO+g zPC-H1Zis`gN?Tk+aEMkG23kPJ+Oa` z&u&`+?pd056tOsXUW;QeF6cuSe&aTrx`af&%VE9qXN zy|f(~bQvltiMtKLR8^kR<>hx;&L`8=J43WmF$m$%vkg05EWzz-N`p##JM)b&T5HN- z!xiZ61-Mv6-2!`j40Y=-!Gl;T!e0uMj_oAw6{(`F`P7X}4l7?cPu5(^6qJW-Ztl$7 zCAin#FI(f?Ka*0N{n;#v-vQAaJV8M9UoJp;6sx^~A8{RbdaMf%`0>s)MKEIP@B(J2jQ`Cxc@19Zj=^c}b(=kcm4#q|ba_@kR`<3o0 zZlo%jd*hh*KBE8t5C#G#j>?Tnlgk%IMn*B1dl-!m@7~yM3dNekqEBV_4w8j=OC`&% z!5Xh();G&>l`N>xVur7wp}}EyXNk-7T-8NzCkB_KBPdW{mFp?(FLfK`HpeBc7jimR zg81yn1QLR*dn!Jx<(^Q`Dl;TkR%SN0>`*i^aT!hg;Jcjdr*~UNO=S`h5qVD75MAgh z_)0`XP17dDXWZ6;wUE{XgKsKfTD_eKW6F6+VYB8Ev#`u?#;U?$w!odmWQeWybjuKh z@1V)LGe$555_kW6CeFH=V=Eb+9mUmEEA$Cp1DxC}81KqwV;K<^ z*_m*;Wmc(w_@=&r`xy1g+oZP+>sweX7C+xW|L}@g<-r&#!d(q^KBtEL?TgdH^z7^~ z6)oq$lC>eW>Y+Npo|N`!X8lzW6dAz{xHhLmD2VsM%BI``{1~fa!Sf7B z$z1G7XgnC6Y)TXoZu>R%i?s(4*O2sys#V%scD`v&cCkJEvUIq>w!Lj$wroO3OiYDJ z_Jp)*a%vfotI1L^bE>{Pm4gm$((|yyQVR<|MMl2Xa$cA|pNncOU+<%pScz>ZUI}$P zo4#yV+Plj1_V|g}d8280tV|kAhUq%&?>2`<=M#pA+f`jx&?c<65+IK~L!$1XJpzB4 zmk?ic@L_ygBb{bxWNW+w%vW!b6DMt4h|m&!9T~$|d-@qRyQ$I0V39cExd2%VC+E7$ z(vHLH5~u6nQz>7hgS`K36oE6ll6j9=SfVTT40fl^%r~aGu;LxoRTsNrnnw!6`)izK zmGiXPJMs{rzP_G}`EHvNA3P~}sRM|5)+<D zF2wz*vmW(h7Xg%TdhJgAi1YZSrWAw5uk$J08qOO5kipbJZ(c21D2cKn@~*d+qh zvZ4VD$S`WQ(gOtjth*6+z+uNlaD_+!F@N#fK|S=4^Zvb)FVQTf*jdVqt!(3)oKocE zR@=2MX`d>#!br9Si>4Xo4r!yiQO>o~jG&|A5_mbtg)*^e0sI33=C^Es&O|QbJMAxb zwMY|zjTN86rEW@E%FcH3(e5b`qjmFw3hVo2Z!vcMc&APGZJ6pK5Cu|W#MU9v5l_hl zmV5a(aM#)1p5COSVfIvAJ6`*npsMGzO&el?WHR5GU= z#qVw02|VkCvH0qNf9g2jdvRK01Mi%ii@rtm+$~QyDg9E(s`B~a> zm^c&3W6Bc5=Op;X;y{U$?QBA)Q4F{hovVXgjI2KX-t!1&vPZPev{(WzNZs20`Y5%H zd-{mXg!^=IXB?-W*QFgJFmCvEMvBTOH-GDP3#{=?xty_UxttKdc6l!rh=2WAgn#b& zNom!|ZilSHdAk?$_ug(hl&C2F$n6W2>ci990cMO04D;Pp=FM!;qzekniCd~8?!AX*fSd@*iO0C1f9`7eH#6YX=Hgx147tR%De%YJE8(_=ZrRY z=73Xcv4EQN;iE^vRh!jyy}B~|3sO_gm-NV!=Y4YW3n{6T!pzz9qr^(In65zH&dK?m zf`T}Q`5Efb__$_@ps5ZOZTA*U`-z^6U|lySl9?IE5v(pf>~|K&^Q`&<%6eV#|FwJfxcw{Bh$rlNI8ZB7*0o>N>Wx0nt=L7I za=oMnyMgWy%G)dB>11a1Trg|9VQsIFRuHR^>DaD81ynLHC;BZ+ILuPcdo%Jom`=U+ zv0@LKwB2Us_rk)f2|Y(ecen(Px2HQ}IEg-h0tHs=4R||1(T_Vfkm}3y@^Ww6wPB|B z%P+|7w0|z)t%Gb~%Ba+!%#1&OoX|jVy|W{=wwA#r0i3pJm_a3{QlP&^g`5g7y=7ip zG}iMYH+QxMYbq@>6B}kNA)j}yrekN!^fuv?>y|WPR}gPnko0KSFs_Gpt2VG_tC0z# zufOW<;w@Cdt6l=Gr*k%y4*Ewj=4+hUcfKg1L}=|g7Z#R!p1OVu|7Ou}le#}|>shJ3 z>6JeEeqeyP1G{@0>e(;1;!8N1whQO50?Ghl-X<%y(g=*|8N$ZNff15eEm| zBe%r__az8lO)Kw5*z?*eW=ven{7GEF^N=3%K;F;W`ymxoIFO<9^*g9I%rt^p6~N00 zEnBq~7;S`~y!oe5S-o}e1KeZ;3Efu-j;pljByI$${#Umz%7KiAfZ@ zA===e(t6uy_fBiT9OsgDr^2O!e^bh1Ute%3w8i=M?OXNvnDt{yDzB-n?d==jY~SU) zRri3$K+H+SZUJ0;O1@>TKGkyEaIDm~Gv8%3zH>_$k^o44RKd%vsl_hIdP%XH>tfjD zy#1`HsVO8nupr zF`S2Bhi*I7goTHH-ZanURJJB{`g1cWDG8-~3&E>ehDg`PgxmyT?lc zypzbh_PLA3Re)*DZFVl^u<<^Ctm-aa#7{7;`FzJy=k7gHAs3hDPcVLDT^Mn^+91_k z>8~OyT9UdnS~tjV7d3wPd%rXxG(NnZEVu3p@`QeO!^JpeMcnRPG+B^Qv~_WPE5?Zuh>Fs84mbtlh&W;bBpFZm zFC7kd_~W(P)XnQ!ayX}DR2>QW4W(vc%| zY+gx^3uv5quQ(%K!E4R6>z#ZB#-lz>k9R@KtKG+h4YCj_EBIz;J)t{Jpyn*3hCi!U zJ|7wIsY#u~RkYl^z^8o=lr^j?UR#(8`s!jj-pJPr-eTh-k4H6F)#$PwPL^ z(Gh@@o7l0b*Jf+TBaYV^?6|2QeaF<%F{6@>|6mjig1emooSq`H3qr)<$i={bvURh* z3d#1!p%-jo1N@N<%uhmFwF&uN)0&PmJyVw#Ddwu+JOmH~kdE&0?d^A*Z}gtEwVOs> zY|mMtWN{J8$Pl^X`r&pZ=jQ5Ecnz=39P*l*?Uzr-x?2_r{R)zL)BG#-J0+puShaq? zK@!%GI0|hx~4~+|(|7x8riU z<<>12?-X1&VXdBl)&Bh!nunsI@@E54p+@ZVH;E^4O6P8Zxs$Achb}G;@jusGqm}W( zNhteXCk}0k;pt;uQ02CLfX4FNVI7NMw-g-O(o+5NeUffH=?SjCmrou%SeS4cKiJ>k z{ov+$YCWD~&*ENB3~$R(P9R8%e+1EsoP6WAxmi$ZPaY2DaROqAYI|0E=FAo&sI_sn z`8L;{zcmfr)5MJOuU+aKmIHFFVAMfo&~1FI%q1WYoRXrKCVlx4C`Srk6M`l;Q4X<5 zBNv6xZ^Lf>TrruFnu>a7`q!2t^E><&p^f)_b<@vC9hbKT1_zm}581R_FRoW>VNrRq3mi9g@L{1xO;e=I&k(v%-pcXkc?nRaxRPip5QgP$@Mcqx{>ZPe?n+zk;ifd z{$f``7t9cNUF;wLdspy7^jUaJ6>0>ne|J00v&Nnj_`85`{foA_19=UNwq9{EJVHZo zwU&^GQ>`&jTDFA2L*LK5Fi6onh1?LiuJJ_T{@lJ2p>@dyL8@dB+Wr7JKBBt^Ex!bTi`NCl1Sn-wF1&R-ha#{~yV4Wf+sbfi-!^&6LBWco7C1RYP+ge- z6OJxO9aL+at|z!59^p{S{+SPzX9Qi~ArXb|T6ouQnL^9%mm~zOj*k8 z8f-1Cl=(KJZ;}1s=DZ%n9k`tUVz62@E&uip_OLzp#>u3+80Na^6k6j-1P9Ua+9VdU z^$;5xCh_yK&P~V|aXwtS9GjE6&eI@|cBm4BoSg8`pt+fVm~d$v;&(kf@!~savYxQ& zPf)FLyQAAmEUKqB;8us0&?7F>ZPzWhMUIV)eWLU0$OYay;j~Ahr)Tp$B_$HDg#xFp zTT_$C5l!EJwNKXj6DX7~ERIKUn&T1>Xi7**&w<*!(VS^4gAJ`IFjS zLsn;!5%g*5!<#mnu??tLlO)lpU`XG*acPg0HJqCQ?Xt&B)VnK&WLNX84sPXEor=L& z6Imr&N@`+v?2ZV=dRU!yoxyZ!Y<2Aw=ZynC8unde1Z);JN=n#?FS9H=<5^Mjj(!^0 zl3*TYM9N?hdfutKu09S&E=Ej=LaT@fX8Og&`EBc}qpFY5ZM6UyLE9l!9pl%`oCVJf zG>?GOD=)_9Ex2x=UCj|VIE_fluJ#`&wh_2kM}_0et&R+8#zp#=k-{_eHwith`vii& zX*lA$;}oK}oe34vWE#xu^UWrWHFV4XeTb@igNedxZGGeRZ3-%maM<*3--ZPDvJaeB znGBm{Yu9;(OzMbfXz)RD;L!nk&o?lqHFWTXtpq2qjgt+X5uxUK9LQSy!U#6B5(_1|Y$|J5@GGkks;loePl zuxRM$Jf@}uwizE>fv3*b@MxLMZeo}=gYhg9d2+-F-%(0V)`h64dpz3_L^Iu5=mUTi zoJ}vJC6cwLuVky2c^(iALG~O3(A}|%7G#j(E-qK2q9$qR=>Ci@C8QJ>l5o`B>)zVOe-B57U6-+OoXNU;>sq%kj~e*Jpsc%jWGUrbhHizxz| zgoqi-tAn4D74a>LoM3;rFwr?kmgAzI-$Wg3_`BY zvI1uxYF*J^DJV3;V0aN79bv4mu*pVhDPl;$4vA>ohb+wnf}Q&dCj5E(2<=7_^XeD2 zwzkQ-X4CPaT3QVJ&WC^+N5cWVw8tn=xX;gofy(-hbbESL?4?th5$pPZ>hHGZgEK_x=sJ=(;Z~!fKt^Tq4bO3sZtYE{4_|IC6*zW? z!4Zc{IHxzKl$BE`RV#Rd z9FQ+m73Qz)SemHQ3}X@_`rx_tc|=E8(($sk+o{DvOU*X@KcOnTpjzd7ewJw zi}!xmtYS+mzU8uun2er96QLCETS|Fx8XM-5HzaE-M^CBxK(A*~V#bq&{Q!Izg6*LG zTetH=$Y9?yN|(a9`mwR>>N6d0Vh+gHLkT5X+Pe2ihrhKoZR2w1vNi~57+_dNG4J!# z-d^C%961)|aLn?}pwAh8|M!0POZ5+IijtFiMK#>zrdwy=71!X6^q*Oyu+=yh08i5JdWe6-vr)E;h= zp}D#WuaCNv3zrt|ZC_V+e)o4dkoPs*_w;7R@P2t?>2p;J{M^y8b$6t!LR zyH@T`T{^n!N*)&F8}Fm5Yf8NSt=xZiLzgQ)b{q@r!#Ho44-{OrmalBLy!RDXH7_Yqz&7vEp9$pLh|9Qf@ z_dVML2+%NU^0#!>zRRg>>dei*w_{*Tsm#Z%qy9b5oMtrpV|V69W+GeFr-0DH8-7;U zp}3+!x~$7?HiWPTb_+!90~1>d2;N84Htvr7=VN)0AN!^GbFaRlVaM7j{}nsBZo_7A zq72$2;_W^DK>=TmZ*s*y{JZ+?m}dJBkB@wcTn=Au;Z)_@8ekgae;Zo+cayzS2UtZ8 ze@q{neYU?;0Ft-nP^4OWxQ7G~4^7g5gN`@PF?9 z(03*gH)WAeF>dL$WyNV_Ev*Bsc!k!YkJ5}L-}`-FZ1a`nESRsc4IjMr^TVU1!v=y} zAH{zcPEbb zKZv?3BcH(4v8uG0X>f6*8a{Z&E#&^Sc`LhFWvwjR;3&D@Qj|2F#5QFPGWN&R_G#({ z_6?1VM3Z%qMv`!|boJ`gSl?>nj@6C4joWrJSw&`cFCv0z!p$dXN&Ff&|JbW& zG(!8q@#uonakgE2q;@WyKmkwNfdJrzbi~dRclTW#U%%Vso6#{AJnC)+o;vu%&Ye3j z@9dS9+i^tM-&V)|(TKl*uzbMEG#I;@`?hf@PWYr_YVt+_;I6NK=fcX(8etA33n18f6c5 zX4|f|Cdjqlk30b(+!e%@ti5H^2n7&gz`SkFg6l^{_6`2p?Ckq7IuztAg%UZ?4z`36 z9h;Q2`*1^5&%kbJD8`(cfr0eH#|w1J6-t#^Xr$EXkbrS1GS~6wUuoFDTndjdqvofN zBjGJv9-ygxWFzt7)}XAkG&N*amUBE({f{YXZre{;GBPt4yq^y4`uS0~{PY)*_l+Q~ zbMx50OY_Tp&lg2~o>U&9Swe6zqCdy2YG6Mr3DHxIKRvwodGTiC4oho_n!F@WTG#*^ zY)bgEHw{PAujJ)Bl~kslLFWsA38tH9?$M$Tw$Yg|^2dq1LSePA_V9VFMjIosb7a;z zg*aM+^lvX~PNs>8(Cwp2j#_<_#Pn`s*w+8z?k%ILY`bV-5JgcWl@0|2l~lS_kPvC< z1_6=o27{CmkS-Mj>FyFXB`pn`ke2SQvo`vk_d7qnKi@aT8RMKihT|FTXS?@(U-uPj z%{kXx&PywUBtmXCpi_H8b7S~$Hh-_t@ejyzrxF}>KsOE!o*Z5*K6-4YpL*3Q%pPzS zbE2iB8G{F2dyz@_!C`IE)&MdWn%unUu#mR8kJ>nT6j1l2aQTk90s{%#PYwB3k|3!p z%5N`Fj}bQs(BgFp#h0I%Uz?g(tEv0ixP_YU>FMc4?SE)R(8Uall08pa7VN@?zmYR4 zGbN8J;ywaT2>uddcBk$0Pq8UC2v*f9jc!KixNt`4INSu9 zcOEcq196}#2nfSL7JCEwZ_(quy24&+j(jH5iR0ne$a%c`mC-LacrrO;A~`j6_6Yra z@lagd!49S8vk$7fX((O$@eRWxCB5uB#W+tV=M)Zm$HLU%*oQqhIcF5LOQ;x zL->aV(C5s|Dv;yXLz)30<)ILE=cuf&XOkvO>Oy`Ge*0Fh(D&U3>PuK$F;6#Xd(EQl zr}FsM58SFYb~C}(%MXw}>(-xX*7X|<8*>`DatiA8-M0~d0MR+l%T_YnB)YlTHJ|wM z$D7C7i^IMFEM=c4d6CX*!j|$nIukQaAOeeO>%Th#btFS(t&@~EmmPdX)}Hd?&pAvx z9%AkHvu0#pyLJsRZ^aR-H#h*Rtx9{XhWai9Q4#fcCN$-i@rK1!#X_&8;8C8?M?iK;^`xPvp-f94l7N2gtdB2?djW#{k})5 z&R)5-2UWvtba+?fR8$;ECOUT8Cas{mX{`H1$_iHO<73N-$=UyC0fdA-yl#A+_h{$Uw$eNo7;%^La2nR`r(0FD$bbrJsb>6~+ zG}Q{APtCU7&AFni8)jku$kdVOnQp4w(tZGLK_Pfa{KBpl--kYur2 z_mpbx94+xIwcGwg#z>u^zqjUze;&hi?}iVBZM!w(l8RbdjrSOmng+7t?%icV=|O;P zg&Yt&uGEYUc0>s0roLX_V@SiJq(oHgWRh#i@hszXp6px#Y}Llru{(DH?f!kn?dxf% zO{IwcKiI+9YBnWqdh{4fz3)q@MOWj-#A-Lj{GMZ%ri_o|mdM5E7{I!`8Am4Rwx=rf zT4`=D&r~V&PH~DoO12V>9D6C*TK9dguh0X*=d!nm$ooQTT{-5qxo{bUEt$m5%*{a) z9#X;AOsInQ>C_#!-A3^*hMcY=5lP~bCijxXgT7f#oPx?u=x|05*d27MW9#V_WO2Am z!W0SLOP>DXez(nyBk1O+pr-a1uxW%(;Aa$yLPz&-vNBgK--dTS$rT?^7$)t4TQ@S$ zZbaQsR9A0@g|RT0 zmlRFGc<`U>6AqD}p?I$PuUe7)j6=Fw(>)?eW%+^H3=YL9%60`M1QG8`4aB)}Yj!Vi zRbFR*HMMp0#Iww`n(AEsb<`0>g1LW!e&H>^0q^T>0s3k5v*!|t&tu?^*7mq7t_;PL zhUeCt22wkw4{x0swngQ=-g#!mW#FhoAj}o_cz2fxMuOvj`orLo-dxrK2q|3mHl825 z25O-Wxs+--61YA}Og;GluiM4ncJ>!>+?zs`o)xq{kbdQW2*$V(^RlCzbfL`6gL>Ie z;n6L;p|zJtj*cXqRQ-2d^Ry&p>bd1jS#QyvCv70QC|JK6J~-c!SJQ*4IrMEut^^br zF~n39l=SQNDb6Bvj5|KrWx9m;QON8h9@6Xio}WvvjQoy6zBgRKuHgo=NCnHAT14z{v^@%+SznOrWS8VkiDUvjc%lV z?-^<=K$#K@dlOx2y_P~%u{k3)7N+zCs-?(ls-zVEg2y%p9)m>hXp~BQ==y5(EfVWV zH+4~#rs%ua?|<`?B5~Ajtji2&{XSyW4AZ~V%dz2_9Y^|=((vz3j4*iLCOFRkO-uSO zUD?ly^VYj*nQaf{5@ykc#eYAPGJ6+06wenKtfyL-&7xXK*LeT&H4HLeq~5~Oo%1`V zVHVNz^BPJRNW?L>Hu8%=57J}LNT&E94>fL`=eDZ5H_wSGU+1>re^}+NKfdTvXqWn;gcb=c&<1fhl=Suw~ ze7~Z0#`D+z0zurx{`cWfoTq;YCiwdQjbCl^v3>09?;S}T!8Oj{MW;^vy@AkOpuD^; zzRwnXx7*a$_wP*y$gjPk(M@~tFc3X1=E3Rzka(Jv(B$IFx6FM>^LFdU^Uewoc)dlZ zg5kk;1O!0vRu%a5>jiqZJx(;}4*eiGBt@qdeds#BMj;ow<*6AQ^%{nOz?OCL?_Hpm ziGQT#Y?2xH7&TlCuVHJsjh@uVE@EcRik;8PBrq!-{uv;Oi_K??qMf`)ZlFQ@pvQ4s zgH6r#fsv(>ETmaV&G{Z`j{^>lg-c_}O%s=#Fa9lt&+i>)joJP4C*}Lz2(=B_JLGI@YkU)etzORiVUyPoG|ZYUZ`6>Akx{-d*$j@s<(iPiGc2 zVg@vrEvuad2r&F!64M-Oo|V_KVh)2;Q~$Yb2aD*f?o~ z&-d3#b=cgSZd^b&6&609?>Ogs0j6|SzqY14i_{{tURP~gEwFfQe>2ftSR#cK2s8R* zDPk)3ZFGzBgWc{G3&)G8lF4`OB_6t9BYwnBPp%^|kTZ_pB_Qc*~gc zXYg%gTxG8(QrI;M&eDVB+pLS8m?D$2?a%;U;7E z0()VBH2N`N(x8-rJ^@QwICRF|Jgn6QG3Lcrui$w|PNQ-LDoe!tlUsN{*7gV|?x7=$ z*jzj9Kyv54xU`7V6?eXa+C&%SXDXSP7k{&Fun#IpiA)aizReW}kfqkCiek#ON4!=H ziB?x7(nK+T@^)0*Y)PDDkq`Iy!;EpRiAT2We!k z6kls<5&?w;eZp{&Gsarg5hg9U zw}k^KZEvK-+mM0MeP6Ft^cdZO@_(q1PR{NCIEP+p%9ILbQf!qZZsk zmCzsgz*cJnM@5yyAudR_dn9#Tc@xAa9b0`#NV*02lw%zP6kzqv`={l;H zK2NR48z_LZRrIOYgVhGb@9F7tspOUzpH;fFFTSVutXZ)bcUw}rfG#ZmOZZY4q?6G` zA7e+F<~7(zQpOc!_4O70Z~=qy@`as0#!`%92W>;u*bid*h+K@`#Ou4iCEs!s?SI z2&{cd?2GSb=h&ze8y1iLl6IviqDL61jx?t{tZhQw>OO{t5D1jMc-8;7B`Ek__nfl% zo|6t8LT4!+%-vPa)&$Xz!B96o2ukFX)qB22cFh}1H|+#cJc#uVrTWu-eXbHgHIfpZzwsI7B3sZSB@hZMgV@T(F~5=i7L1 znRt*%tlQ2+7qFVY&ra=#BW}(%G%~Q6c49{|sogl7d%%0on>?qI)E6NsCPqyEe4kZG zDI!aw)(4Wq0|d3jx2!C$KYvIZ%({$rVj}`c-RVH_2UI=RA*StY#w<###67X^`9fJG z3@_@(@qulhfD&pe9HE57NW%W{^}09y9UgCN{o?S)(}Oy&^U2HVvw3H$IfdNp4}Iit zIzluiF*|hc+6|Sn`$lOWFhIKv=vfpVQ5}($mE9qe$->M$Sv;tt+x_D+Ckq#60g37Urp?A0r7uL=8o|9-c%jXbiiZ!j|^54H6bv7+L=&%%DCO}@x?YLuap zlSF|nd^l0peV^({Nc*##HjqI}S_>Nupb;UXWE_*c& zoNFk#SU6JWY*1?dScr`K=#`wjTHU)O-gB5!!u50HX^+GY%c=C;)uXpZbvIY(9)4N5 zNF!N2MYi(V`tIuwFq9tUi;GKrw0$$sP1yYk59%&F})Wk$*=R%Cz1 zJ!KLa^NDz8cC}i)>#6Bt@6Hc?S70lGE@C{+2v}swjCzn!XjqJ+)cR%GjqKdj^>(~UOV8O;;G%5tnCbjuG|W<{&lUZu zU~8BxA&mK5k~LoUp)D}KNc(Bdzlbn&_ZjLl`Gv6+HeEH;Wz-Zb{0|o7Bm4MnUOrTWCvUx;-pFBc+>tQ<+;D%DKo;lc5A=&x&#b&265h zid(4Nr`QaKTwII$VrAy63~yX-Q}?w)-63qL#Y^sw3+7gfQX>ET z0s!F!Dv9@5FPTa)XA;LyJg(gkWXjyL#YX7vH=g)CzXHhi?y-PW6wbe2S%O~?W;9MO z=1;@sN#@@eGhoghj&IP;-!95CF=f*Y<5baD{F~Su6Tr3jzLQo>OLZ1e;M-8$brBPzd0{{v2ZRP z{J80ES&=w|Sd-sNnTowD`s`nsNhW3QJND<9wBksa{=HI(rF;v@x5xU_&#AM9+7DU*kbE24`h%)VI}+@omt05_UIVkcv%aG623P$Oy0JTdjwd0Zws6D6wW zw;YF+Sv&G%Fh5?v|s)qdI;ZSRAxwJ~@DNr7F40Y=#`TfuxqE6C2(-Qrmu-*^eW4DPSl8aRuzTZsr z_sJl{ufH&?JrG1U5-N-yd1xA!Uh;HDNt>vqI9B&o+1lm5jGROzWIi~46NE1pT$j{c zNqm)l5WhUlWZY0Tbq+j#S}m-hka6hp6U`QG_DV~~#+?#e#GqfQMT(7QeEnZpyU$rC zMvTK-oA5o6$cOHiGlh_DavI0~p4ySFGs%=4UpC7<^N*w)YTLVHC7msG97s>e{*&5~*le~jzeqSs2lTbXv)Y^<)GJ#fy9lNZW7vOD~Q?D{_ zxia~8O10>vq6p1%E=Ij>GlZ)IIKgLZx&?5}?Vj#8>mD6lxBL6eFVA?@UD0|@W9ocT zlV|zwlb#&C6-^T@+5NaHB`r_cCN%OR*t=5Q6`)MitLGh1=uCyC(hnvkgrl zY|#CjcmU$@_dRU8E$6*aa0d$>@vGwYa?nQ4C7_TCa zv;Hz~_`FpxxAc^X(m(m*w%_5RjRlU{PMAL7`-~qEuLGcr!?uoLg+*^Rq_bWzf zNp|*J05%bS`!ci3UoeJdQwo?r^-rX5OD~@NN>wm(z^H5s9y6g8=#&acUh-FGcN70TcV65s-U1y zz7xssw3%Kjulk~Us+kzI+;}NKegSn~33gLXsJ2PMrw2TH?l3sc zLCMZ7R!fFdlZW;rbUBA7_GM#}WtuEdvZbo+eB#{7*oubIPL`f5u!3X^#7VjG%&_M` z0g*VBGZU!NK1D`AlJxxQRR_IT`uWqt3*ayBCo6I5!EOK6)~l&q(FC!Rp|P<|55&(t z?3Z19z4SVU-;qddSVM$e4hC5S6L-EXSLhDj72U$&miFaNe1l*bu9Du>=?>k&oTh04 z)F_xN{k`Do?N;F!K68b;R?ti@ts>QDWQ6Q%vVj$!){9RnUQ{~j${#xNKzF_wV2T^Q)S{vtmbPYMS85j-o;j8qjLYsU{#dZ_-)w@XOJvl!qt_3XyF4*K(Uz zaD@Mp(#|p^krK>Z1vbdiUJXA~CmvMpIc3YwK+Yv)Xn4sVOY)7s38qL05FjL&L(98j zUB$y@$8%Ky!x6Os#M&;ghleioXX}(ub?DfXV09H5cwnK~&UeoAb_oM;o3Dr9qjLNG zfPgTJ&5R2wgq`Px8K9*QXK1K=yyq&a>yx~KdY~Rjyz}8O2>6mJD%8;cS7@M|n+wPl z(rb&9ZjTjAux5x4M0}C-Ip2wDR8NVsnwRn`Qv6z2Wt^ONP%H{O9BR*Fp|_9F2npF@ zJ?qanWbOG9UZyx2Hf5zd30zOAoSikCnRyB0r3)flMYQ{jj98fD^$}Ui*?lhtLA1yX zbW3Y2XC}{DRlpx)sW7tCcw=@jNnUN3jS*^&Vs4B0u$ye|eVI2lwxne=WkkVCOM|&k z3UqEU#mU+4>Yl)blqJOTjK6L`;_4hk2~`LrKEM8)gqE}!4Q+3|W*9-NMViEBmxx4z z2t-t8-XB&l)DQxPLCQaOm&$d@qqS8Eecefb1T)A;9ctK~_X6R{nGV$y5XA>s;KtHB zh)=80xn8}9!9#%q;y_RoV}p=R7dSrNML&b|tc;oR`J}H|RiD0yh(p=ZbD7oPZriPE z4jcR7xDUXxLo9^vouUlyd&pHcbGfyzUV7+i!$aZa>kG|jdpUsHSuUv#$xXK6p%|jv zRW=Dk{yKR+Q>LP4!R=9-^k$-V?Xpy+jkovY-ocaG78)}fPO&HLH}FFvQN@obmxfnx zpl0!gSct$o<3!hYqY33-jxsXtgRxO6O5LbgOab9+OH0Gj4$Q~#@4;R3Y2VBIz&(iV zz;_L3JJ{tu#)h?ih-ed;;1d%SgJpIF69f1wd_qDgNG)7;|8zi?%IJso1dJ-o3_ypA zQ@Oo+{N9HR#A#4j)d2)6lmYq#^TMryl(-IfEyKyL&f81;A1l`?l4ONWaJ{{yOUHJO zPF^S}QKmW;xZCnjz*sIW6=xL*S{+sGyL%j86&a~ISH&Sy5pcMfkfU(iqR+SA``WB< zdcxcm`e9MpTWaZvRz-z7j4a%r%9Q2Vi@$dF?$A?nb4nBr;vF!rpshXdUFO#BdEfL2 z+nor{-(J(Q`tzNd&{F1^lhZ}TR1fN_BEh_eDrTcaiBRUr!gHoyP*|cMmlpsee^ZDs zC@Y%gI>;qt76bhstEpl6`~S3CywMz6oJpo72anA5)R|a*ZR2BjVFEe^rlP#sua^Vo zVTOc;aY~ADEEhsX%e><@v2A&M93S~n*?Qc^NZ z?rMGaPOECYhaD&|8Gfiz!=d)pD}?2%UURxO|AM~YY_G1j+ho^n)e*VE2wRUUi zO4E@+tp}BJesBB;E)nux31i1LiSA=hs9ZcT_^d}Hhxr!$=pcT2xUj#!)I!M~cT9HT zsFHdJ6XC#g~Ja9$y^M_6?6wXIURP<%-FAh@FfeSCmd;gP*q5Jk#y?w&x!QkfnRMsMnpI-|OkQx@v+bOPl2G)G>z!oG(hBx5{ zG-sp;L)9DOsSR_I$1b`?iCIpV(D{>FwmdAC1+t|l-9QD3^6zh!nnAuCB7r+D4Fe&$FtC}1HBX|$@)ga~44gQV_vaPxHp zoYRwb6%up=UQZqq_=^^p_u5&Rg-k~aIud=6V=M1i+nO92(7t=4z98$pmHGMT?RX%=ILRePsM)kq*o4A#que|^ys$L?)duwr|?ZPgS^u6SGYd}i?W zr5nKmx(%m-9+D_um7|=Q!e5sR(;~_>>O@v(fg@P31Q)76h~eRUHILgx)q;b#JyK_` z>r$)ulHlR~v=`!bCz&19{Ph~Yn8H*rvmh1Nf0B4`G!2^jkW}e7_U%>Q7$MvHP^E+U z*)+;PwZ!a!#_Ioq)QNA~?5i{#!z+3~B=~>1AVD-syPWmQybnod09@uFGI;^||5)gvq3xY8j=2lH=3ecY~(*0UC5u zmv=oH2{G$11bz$@uL$lN&+e)=Xc0x)|4B4xf7l&pgCCla27BNPp1e?h!}q-S9Fy1~ zE>-15le@4sI6TRmotNHj;-0`%1i~dlCATCwD4_vM|vYWm>z2fbh-k(aHb^S+sNLbQURCPYPYrt4>u?&dMJg%iig8%hM*7ahyE`5dIxkKo(2 z9hfz03UP6(S+O_kVMn=q{wVw&nV{uu9QB4Fd)Oc5n+J5CWzFfUas7Jb^}4)24EenG ztuS-Yqgx-dcnh}huzNYc$b|v#Zry_2=2HE^@k37*st{jrb+ewnUb@j<7_!xNTyD2G z6p@s63W;{9ztueW<)cp!8Ok>UKd2GaCp*b)7yZs#eWR|J#}m_+*Y|0=3LM5>zaGr! zh-8k75p<;&cUgw*{>`X+v zdAe`UH_DqpFOwE-rK;C5`D4izFJ!o3`tl*e2?nypJgWAx@DX9{WkGF*{6Al`*e`C^ z5aaXSASqtrERaS#zN@J=CtD(FH+96SoK6`;D2c7S*Fe+40tB77;_n>DMOz7!BGgZ!|A!79$ zo!;pF{Ft`q*hG{@56bOC=*VVkzJ#pz_N^W4B7apsf;y@ zwBB-N>|e$F6i(IJAky>>Rj_%r=_JsgpO3*;F9tCRGn|Kw+0Q(cRbUd{Tam*S*^TmD z^?dgNg$y-&og{lq1R_Xs_GwGyET3e463nI&y_pkeOUML4A4i)L>1 zMinEZrqBkjhMtI{C;Ry=Gov21r@}P7o57DsQ2}0vm~* zCtLsGgD?e-hj22zq>h;jp19$Vqa;@|He~3D1OSY&;RlD~@>#7tZV9m_v!%$6*ly#R z7NS*i`eZnA`E~OSdx=)9yQlE+Y4yY5-MrNZKZy_LK~(`R8JAfDM9E?ZpCBkhytC%M z&IRg#+x(2aP;Bi7GfZh{f-Bw~y!cEC zZQa=&oQ*$voM3{EMkj={v24S@q4E>#tFZS`rzP-RiR%41v&QfcP61Jx**y+M#<<5| z81O)RFXgipBck@$3y!PIym@U%@a+C{$H1!gnn-GDkLlgt58u1X%y_9as#9>oOYzU3 zn=1D8ISKZ~?Yn}wtO`ZryJ9%eQ zdN&SwRg3lqukN)@T3NXzodPz&8<@1u96-dHV!Wr_m#!EB$MB>>W%*$S`FEY#+sR_M z3=~(bL9CeXaiY#;){bn#@p`!P+PF}oNZz^R2Y9kBlPu+u;n!%SPtX9|9cpeYS7t32@d-+#fP#o9B zujVy{7n%()G-X>QN3Z%fCFV{Z`cetHYyiT;22&61;xi0Qr@C*?4=n`XjL)3f#%1); z>t2aY1WPJ*9k&B4IP#B=nd3djpY~-b1voLuWJZcV^f)ec9vP<1%c z1`+zrwBDydi<}GKjH;M+;s=^W1_p5a%?3PChTk5KkUn^}Ei?%o`YN^B4~{?CS>&{; z94oS%@b^64l7y23jLUApg5@<%>SEoRSKf$#B=Y^^OD~{0;eH*5^y`7|k{+i-Prl%x z`HRL&lXJ`ETD-qnER5FE%l-$96KYvB!8YJ_OpF?K_0i+QJz@M_4il*Khrm&e;BHa@ zn7%y%BUaAWeSiSTGncw}>r@ZUXVdaCL~(c?*6}YORMpk_Oino8mJ>e3cL@Dp|qQ4)K+{>t&8CGVcwZKRvYs=hxu2l~j)tpPQ@AK0!+oQeC%T z-+;izuTkG@l3G!eQam@plz=&(P@e#tYBd&T6_wKng0 zEse{hCy>v6X>*Prffff*jJDGP^~Gl&^3BO}2b%}pW&dRLgjps^^R{`qQIF56eC+1y zKso{}CPsLJSHjrM$+moyB=s>}L2}bG0U{~wBRDvzJ!9LU56~l;#9N_$z2#fK+zy9* zMU&-;jt{Tl3fMV1Ux`QvLay8<3!+DUcxnKZ8vrs|lwKyUtUvkCsLw-*BVfV(#p=)J z#G}ckNvkQgj;lLX1j-lMN?`iJkJnGK1(9(=Q07eH^R{=gVQB;EmuGFL-1e!s%t!nN zmk^-a8C%d6{VES*C6h)N)#i>;aoYp#t_N+)@q) zmlihc!yUo50Cy%r2VA<+`PpnKAq$c-WNd;PHb$Tz_7n$p|DO?Zq{^M)m;|@1W6yvO zA6nYmQ;fT`hq&Ofp$RaItpr^?Jb~7|2}Ct_;L8A4H}hqa!ezPB*ohPGhuH5Sf{==0 zooi*xnUARzmRV6RE?@BB4{J0f^uGwu+r$>SSxI$T;8?Ba^2%DZunT}2Hay*^ggo?F zzN9J^Meqla1?k>fWFh%mPi|zZQ%RnAt6|gcv$Ci@;j^gSa;b@X z7%IhbdQB)P_X#-o8L0VRmAW72BUxV%M(gk+XMKcC+5qXUoR6F^P$Mc)+Bh`Fjry8T zlCK)LH{ls)*cNGZ#Jp!!|0evvrCs7`z*PtHb|_6#nbeM%E7|+>wEliAHGeHjp!v}v zDrCV7LMb)VFSZrj#k!#gcK9>I%)NFWLlYZ&45jkS8OyW9L$pBylJC~oBUn;uZz^Q6 zumIr`Oal)cGVNUGaiLH1>+ahgv|+XKpC>1$c2oX5b-KfE1%Cfwtv9mL(5o5L@!eO%lM@fcK!xe@xloGMC z2Yj7+f?BgTzWiIWNUi{4Ryua}IA!LL9~{!);xE~uUJTAn@&&u%%F09ud=7e}`#wx_v? z?y5z9RKW2S=dheQSbu{AGQMw8^9J9wt1+oUqH?q<@Ix9^t6$%2yweCfJQS`t!tFrP z8%)VZh#e`fPWEGIrw4pSM_IVP$dMM#xBvc=?D~7}Xv3+>ZnZB$I*vQvdKwc#>&8jFE(uW z;*+BUiKx!z^|2W&G#*MB)NR4>Ko#;}p>hMF3mOkkG8y0S)iD(L%7 zEo0`BFM^WhsSCfq-j3?Y9Nd7r4ogg|E1eeGn)T}{k@kJSW7@4dQe@F~R((tX2MT>s zTtWiEU4&Y1^OL_D=b;qLt|;wem0+O?{$ppDkbRL!a6t*x`o7Cw zYkP(lB<6e)be8vfP$y&c0)!1j|Tp$MlA5v@r^d)zpT;0)rCOj;%nK)FUZLMdysW*Z9)}Be=P6s*H(3-yd8hBzh!-#EVa z`=+3l6H3YOu6&}K0Qm88ZI^Gpc|@$-UDZidr1-vSr4*stKk~aFR+xUezLl4w< zExj0e-G7Wk+2rd6_A7GXV}Dv1BQzuL<;c&%a9o2R3zEGxsJ(BSrN1%O*0s$qTY`{26vLzs274*>q`~s4&uqpjr`s$tiDklML{rvA2&OyS^ z`^u*=xno!S+K4K z5mhd6?`W+j-}%*~*q9+{$gl{p#P3sj{W!%Dfi?+PzV%)r0FoB@NkrsBI79c0gCKh` z=NtiUp-%Ziem?9#7yKUy=*KIKy=|3{l4_pbSl-x>pnIaGwm>b^DNY@Hi@TI;PuPk< zG3|8U7m1`6pkf8~5MnME^!~B)AqTa98Ktsb3&hs#SPb!1&CRtZ4jf&lC;sn;l26te z2xY$+hm2>bSBDMioH^PYfrV)M8+3-PrnubopA2g%dbsDK4v*H-SBa!R%i8)jCY}p z*?MmKlb%?ZZ5b8+5n1>a!>5MHJ7SA}7f9u%NXI!iRSR)hv>O26=ytqu)G(bt;{jS` zOz>c&DmESb2A(?)CIagxt(JQd2bZQtZ9CH}C*DHB7^TbB0y5C^z0_sr8QLMmV93#` z>nF&t*z6ZWP_Xb^XF>-C?zlH(=qRm32vzlWwL+p_s)666h__B%EgzWfV%ykeZvFDr zp${1ZXt~Mb2%}4Ik0ZI3sxG^99P#FqVRN&sYpZcx77cRGL_t^fKkc?A16h|?qT=kA z1|C`*lZXZG2TIUg_+c2O_0wB~WqPYixM{(zCsTzQ9F$R5>sr+oAEZ8lZycYTFJRG` z?#(o85!C5ydv?sNof`$I9Q2Y3b=e`aIxtj$qu^k>7x&HP`c!1xT78Lm3DFE8mM`!8 z@->HhJ+DbHqco3+I|UuDe3APIP}awU1QGB;TLd)nIBG*^<1SqWP?lq^z!)rV&B0dMn%h6&>|`hWGI>Sr^X4K(GY3@TG_l{r;w|S zv0AZ&Zq$(oT~j7cE7AY0M&G>qSR&wy6Dr{)N|WDaz{^{H^PTsY$!OWVy|JSf7UbFP z41&a`j#FjA56Fj^7FMU7;xk$boZcEO@kICCr!2L{_vvLQq$DM=z$FRucN?qlELid0 zhEE)RFA>;U4k(Myd0CmQPTBbR@q5_8N3G|BP^V;35yxuPv25eA z^*rH__KTcfe7q)?`FKO*v75HjB5Pnu4&-y$)R_lHx?j9H$ zv!Uo_`0L4>J&D`uCDq8rJiRmhneVJX#B49x3*ZW^^N9FpLINPG59T{RPgJtCe`4%S z`~)E2yL>XeW%U6*L{Z5htDU72!KWPtf5nu3$7(MmNj6JsPYBxzoR{;D!s^7;)ybl? zZSFKv4n}Qt3ZHN%fj5~HSbaH~@BQ@MaB=6b9RJ`X&A^<1$Dl02 zAqYu}c1>%Kjt{oxH}?r~R}02U%pn9=SXcmI z0hQ}^%+1TtJNq zQS1C$vsOxX^jcuJLw9>7%I6m2oK6qU@b2FGVhiob;8NZ8e|dWD9YBbZt+BPYU#16N zDqnNFsaPMn&JvQVU3e8)B1y-@hW&f5{>^4HsFNCs7KGO}|M*;Yu02@hvtQio7V$)^ z*g%OoB3j7({uiDyphVKsKh!2F8kT zfxljzJ9QH=2Y)t4;#BR8&o@jzc;5S%jsX1B?jSXbUGAJY&)$I`nUAG>|0-q@zn5q| z4Ks5Pz%Ov9dKSvU;Jw^gRN^on2^`dR2cp^!rZxo&wR~83LiniTNM91F`YK&9jRjG= z9|^}eH!!=C7D)H`TFdS-Yeg9ntB1`ytKtGww9L#2a(5d7ScRl+;q6ZozxX$gWmBJt z`%v6{`qUeC$m`gyJ33&H791G(TscRx4!jte;J{3XA1JU}(6^{OaWQI-ekMR=fbbDm ztv$m546sxxj#kXA$$s$68zd>{$P)j?uYkKt<9=tQ)|OaJIbfYh&$ZDSRju28tnB`IFMQkxdZLE-hzP-*O3A_=iiD8{NRFN&-iKrK0kG+}O+FQR-R?ynjfA0Pb|eh49scOFw_PuEzkKX|XdRLOm7 zLPdrlV}_)^!E^^dM8*D}URU&`FjSXAHtu4C`F4uH44KonwP*Tz(u#)-+rX2nyvwYO zMwLho@87At$zXs)p-M=WE}HwMwLZ5~onLv~E-}Zy8mV`OKxyosco?4-cusnv0R7kq zT#-{dZ$bE*sFl18QcK7wGj*?~IRs zaNYAmDEx%U>%Ss#NTnj)llkyGzr3^9-^?Ms$u$G<(WzBjNpZDm#()2r`(^vsz~<@6 zPlU45Md4ka9kNF}%LE}^z?9`TUKXVQS$ zdraF^UgA8Xj{D$_oLRvSKVsttPY78c`Qjlh&nsFH0B0^;(l7yn6|yY{*ZX5W=4xIX zs5`p<#eT^_;G}fk;b^G9P^!%IPlssouV+3IQyG~D__2nq}8YjBE9kw;z9Nm&#D>YEc)drbO^@n0pm95FNq{e2`bo45gui)ORd^gaW5 z@draSrLp3PeqdySm&T$Xh`CexEzKDA`E{53XVXs%IUs5ir*`>3HRcM|ozILqqs4Y! zbC!d_P1(*|r8OVUVj%LwV!VpijobNx=|C3B7FJ-lg&njZR<%@!`5kM`wHmt>o98xh zy$f`J>@W6btPL7`JLu)fanuXa(!7jt)btcLhvK(}J~7flI#Xr$#BF zf>*zuz;Tq8#N~&x!jIk!F1gS%-~deGRSWF!4VvAhwZ{)Q}m;NcD8!)!pJvrKy@MJTP~43hZ`&9YPCV4cDK+-+IZ}+7&=|| zdBJmh0A+;*g*MQ^?_P3qU`mjKPC1C2`5c#@xQNkvduyQ3F5t;{7F)O|kXt-x57`SzY)6N`E^#lxdVJ<5eYnP#iR>x-4LUyRQU8l>qx6f3~{O3x1*seZo+w~od^b8%RE z3O#PUYuV4Z)tHn}do0B*$OYzO7<4|11;qjb1NbdSVajJqOrcLl{n8HfQaVd{x)mv! zj1LHOUR;F=hAa=tz89ty&>eXV#C!1WxAj7dEXov&hCj(-ls9{QiY{A=aB<$%Cm&(GkY73?p- z(lykXT> ztLEvSNnxNoh=5xCl`732{HmWUa1;L;iiZcwdCE6%{Tm_1XWoNLv;_XYE7onEU1+>{ zIBip2b~5k<#nwvD1tKPAKGbR=5Fw*&%e?HbfFu?HM{~wsUb|O@vLWnS0LjUY-^>!Nr>PL> z!onNi;U>cZ2u5v?#V8ecT5sc+_vg1{dsC$^12}^F2KFsm z7hl*%|gf_GU>}kBDv(S?zUel)L_6W>Vi~gjD*^ z(3dugTqM{$OF@6rC{4FZ(A$TcS%U%{jrFH~#XIjRPk zEh>BKUHB@GXhO}!3vUTlJ^xr()Gmpk<`yNtLn`ds&PkE-wX*(XL#TfDO=^XL{{e1>V$UAwa&CvK97oP!1 zO)5O8VNt%K4wRxgiuWC>pp=>xlRj8ac68d_I6_ygdWk=%>B$S0Q6sY?K)240m|CC` zPn28_<^u+;xclRPxojTK^~RgK<^x5%p011dt3wXsKwWGLo_cAG+yJC=l%go~HYnDj ze5;EFkB}u#?|#=$r=cnEPsL)y+eX+V;5xsneP5V+C8uO+gFXdULJh- z#_@f^?{$AhZ#6O>?mdcD$N&y$H0-lY!jTe_Bs0?=Z^A*EP<1enH`5p4KM^z!l9SJX zN<5F|A$kL3U82j}ub0_j__V_F&@tgF@MeHW68j<^-@?1a&+psb&UE$KwJY8Tw2SWz zFJ5@7)!D%z-AV3t8%jBvUM9n*U3>piTK`&eQ=5+0`!lt0~^XEk{N z4U(qdURg@fkX2}Pl{S?I77~2GBwo}Vh$n?U-5x+Th7| z!j3|En~>Qp+HnAoi7Vvd%W0~N&Nh1_i8@6299LUGdBhgDIS~n41js$D32XGgH~~(6 ze1Fj^7oVZ!Wi_k?%c+&P-GggDl$Z-qYqx;+Jly{xJefWdVVk`Gm?od)%x*OTj-F4?a^?rZ7&J;QOvIc(hf z+0V1qnsfeQPH*I{fFrBzxAlYDpCKm#dhcoJ6JaBxraNM&zYjXh$Y^D<(xqPAS-m@QQT9^`0ZTVJ|F$x97IC>vjNlPa?|mz#O%Wg`>( z)85=MRv61>Kx0Y^_8JiM0v`k@-|pp^!wz#&uPguyo4cUIP(#__Sj+0wn&2G=VrVc| zPJMDOGH#=J`XISU`J&ep^o-%BqK#p9Mc&n^Glue?p@I|(u3igQ)Eet+)7)r|&F<-$ zuAUwY>`>CeY6knso$D`ucJ%zH7%-IT9Pc|`XXlr)>Gc*d=uZ5>*w94|0>p*Wy@j(M zQI-Su9e$;~V@z~hAvGSNNebyd9Tl+Ut-e*L<1C$*5Lh2c;?h>-_u@BcC~SMsjl=$g zNo5jDxKbNBo3<%x3^*}(%JA8{y#|-)R32Tc`(`e5tW_4ug=-ZIW5hrJ4af-)wU)0h zRwW1?J?Y_Ibw%kiJN&B9J+7qS1zl{Sk+B~+a}N-B=oI6)n=qnOr>_yII&D>7u2wIV zTs5#d7c$}74-T+N1_w~tmM7FsyQFEQQJ_2$<$Q#-h#4%JC7&~YK3>HkV>{vhG;{aU z*%8=cUsL-{Ht0v{JwT(r4UgxX7C#uQJ9bg)6Agxu5b@bhI`2}8@yT6Cy7MWhzwO^A z(71XZm9A48*aM&EKkkO+O>?AbIxs*e58*3jC-FED&~V*m27y39!^s5$X`=-Q-zUng zY9?zElZ?U(bKO*EhJnhiSEEeryA3>cdREpazzb(dQG7iBWgq8}hL2mxm zYJx?qK&5c65Ge6vKHKVxjE4TNyUeA=PEUz=ZI_aIFM<@+&vD?`+x9yxo!WLg8CV@% zh~Y9KMDAWFbt>m~Ot{WVOK65tD6Rj!uB{>D^T>4k-!knaML)zlj7 zgaX=GXtr0UyET89NiLRnmIzw}MH!keIs*X!7|1x)&X*E^G35X{68$8V>pxE=%!Ufzp3|-*FVy9kTU-u!^ z{ZCMnCbRbHfpFAyfB$wp5pQg3aO_T~fkvv>p*mErBG$H&dZHFN6Fbd>k`%AWHkxm`Q} z)_-X{$09=_Vu=o>f!Lez{wkB?Bz9U8KuT#`&H$%>Qz65jUpudoAr&JGtWSG)-p#dW ziZ|6}%`Y12uNBe9ZFAM-g(SqUQF8P^h&l$Zg+M~AeGYXfCagUkVH zzXWQB-9gMt-465!xWySFy9$JMx&+8GA2NLT9K}K))~X$NBWpGtJ(JuMpCwf%DBB#`$^?(-GMTz?1D;4DiM zav*CY9LZr+j!~X;+sykS&?ZAv^P0YbXDE9p`CUaKoTEW~8lkfwP(|5lLXY6~*b<4i zThatkD-8ePKx$Wy1g;^!0?hwF4$Bn*To|-}N|k>U0YLW}Sbiy^Ohdx}2fFcKRtumj ztPqF3Y*2p2(PT&UCPk<{w^u;xj^#ic37iAUGC+qe2+IjD9)JjZSY>#FHZ!Cqza}V` zWV`{1qETjifSxkYVfs*JL8D=*Xg2$#6J*L;$^EnuW8!tuTM#etT*44BVET+^^2q?Q z#RU2U#+7Z&iCvOE4A=sn0xT{VYbfYR%g}%{3qTR4u2tGLZfD}Svn+Phsi$bu|NRa3 z`JfOqmt-*;@6A=NiT(KT;}r2}aD}g2rVKwI48m<_YDyLE^d`gk8Cu?=AS4Nb-NFJ@ zFcehMkTwF2ueY*C;$URZUfhBNb>m^giRyNgNugN~^o-iwaIq*h9;zSAr#gh8A9GAZ zNUsMX-tG!0Z-8Z1pXPxkR}JlbPY(GVaVn-pZ4Sax)f%pTPzrj?1s@DxXe4dW}V#%JxHMF z85=f09ruufenIf9P+Z;~dDabsBl9LLr_A<3ZLQ6;sye zhO6)a=oK+SVY-9*(`ApseW~Qk?!iYvT)6G?A9y zZ<0kl_>23`!FA3}i1&|~z>wr#@Osz!!Kav-Q?2hVuhr?w-m(Lxx9ADRbI9)>28rJJ zyixtI(;n>MOR`PN)?geuzVu37DSE!V1P0O)DzEBUVXqPRWHCP7>UYlz(9@E zQpSeAlaS!8rwom_6}ZaV7Jw=M-(<=w48JCrk_(d<*v4)Zj6fMq7AB6=pImyt*=)!b zyO!3A0GWX11P73+mRz6Xd4SCTMazT&_Z#rnJ;W$Fm6+lA4u z-#2Axlb&c#_>hGy@>|yMr)OqvT5DX49OzgWC}a1D`^*w2F4(((;TW*mBFAI5sVcOf z&fl$@hYL(Z;7$3Vsr=!D06w?0)K+RBJw4qu9gJz%uwBB#=b@;@4kA}f0x`*2E}`}| z(FDL($bM}ZAfF-+69;M8oey3<&r@_Y0LWsZDoE4DT{54!#@n!=-AASOqj=Fo@6_(2 zPgA>gDTCs1Hajpe{WFe?z>*P4{bj4DON@?dzCgHSxBWRjcc!2P8q z7vIlcuq2Z|DB~XKhyOl@Av%aB?&E*|E38J={(oER&`NzV`-`NXU#_D$Rjr@DwmPQr$JZ{(w&wiIkScC+oAetYYoG=^h zaLFv`LK}CL#>R2W?oX?LMpgyoi~rA~Nsx5$J*!?ZC^wt&>DJFFVqhzMbmQ0wTIO9# zHo_MRZCzMh4N@=2w0gy)5cHw;Y_enDFr8R7L#K@)Y7hUv6=LP|t>ngA{z?9#aZ^Jy zKO1GhJZ{!q5s_oAs?sV_(5#q9Sfzi2U0dg-`XK+Gd*y$5g~vN4d06qmr2TEt5{KY@ zicrz(6-3XU8ERn3JNS|f@@Mxv(;P*?XSb3@T#>hUnzb(<&F89SeZl&zq#Z*f4=>jD zW}w$qszB@3{cIJRs(&AX9D#YF`(B+5$N8q3xMPEuaju2K5awQk11Z0GOo+_plGp7^ z?z+B&>86>?^H~FiqoO9~Q0-CLnqLIH>J0;$d&#{X0*#be>o){18{{eVBe*ehhP<+= zELF^3WUg3ptr;3Ze@uDp)g$ctYi_cm9>?`#$or$Ts z_8;Wy+1sS$8au{SmZq<(iH%R&VTlOI4;k}|v{1)335kes$1bgi`(Vtoq#Kp~;qW4R zx1^Ovb@RL70cj~>>gTOhYUecGrJL`Q2iJZjb5;*n{j|Z-ZSl@->J&t6z^;?vFAZNQ88uUKb--pywTLYAhLFAe>XF?6bm{k3b_;C^uAIuCIwFgMO|GWUR;uYl^LD~ zR*Q3=&aGFIQ`7~4lBz1mYlTT%=W4=mk;rS6m)~>d#|~;4N*Pq%IjqKbe#R@p-L|!5 z_Y?hQ<8;8Okep ze#MY@KaBz6X7hKRW3>K>6+I{-q((C=jVB*dJMUw173&eVP=@wx$KovS&M2E6pIkvy z@u(fL1h~|v-1Sq{;NIlaM5$H*^&?jyDh z&#BiP)yGYYV|M2WJaMQ+DtUCOY*BtB&R(Q1@|&inAo@zVU;GwB2@en1h5})$c@*(e zAVx&<8ODUY>F+;sBlczg9@j)`f)^g)5>A=u?vt9@Xx!7L^Gye%i`>y*6C z`;PG^9Fo-VGFr`-4NlpXy!V2E*)Qbpzx}*u>HNN{!8Z*>iT0(X35I9tpI?=1?VonS zBUJ4!6{G9CUcG%J&yJFuzX^ObAF}MmWHzDdk+x?jf7e+w(XnhhMF7{P z)G*tym`Rjz=#PHq_WG(9Ee~tLbo2Mq`e!y=N~{R|K%0nV0aN_EeB3*pONAv4Eso&_ z^eqi~7Wfrau_kT#+MS%}VKIfj!YkUM?*vnR^u`aFK(l+C=Co^iAJ&-AD0 zZkw?hZNBVje~l7i6vL2?7{2=y=$~1;sNN_J1tgOGmjbDM#@*&$za@BsO1IfOt4ILh z_*AUCD^0q=?~dsZN!-`#o1Pra>qqf_?z2j8AKfGJtNlKt6M2PZ`n_490#8=|W;A zTvR2-XK{fZaF~^KZDeNDU`Iuk)~$+9)P$r`Wn~9WnLF4~!8f}QgFAwMRQ`CgnG{Zr zBITE__9==(&0v?2QQf`o`tG&*XL)wh)|U$NXo1Dh5sE-Nhid8P@G)}Bqg1t_Yf>{l zv)Fq~X-7isr;s{8AQ7tOmzq`S;`b@6@I&|{7E0~e^KVq`bHZKtzZj&tF+_$2zBfF2 zMD7INYSauHp}3Nw!Tn&Lz}|%WCMO9CCPzf212az@!I(r16>Z1U{a*#iM$vrua-SZx z)qjcj$zM<6xsysq^V0ek-c-9lxp)5=CeG6(6^*#mWbRUrvkK3LSpBMR9uet%B}WkX z^cg-M%6Z+4W$ECMSz01O!^<6rj9f2{dH4( zEFr2`uikzb(rR zz3Nm(pydgxza#QH-^zalZ%iqgiFda#*%NE@Z{OPU(N?{FE!(PEiIBvy8_7r*-KjZT z18SyoNb5>J>)(aTAZcLguZ9`<+QS1WbA0D}=cj5dw2U%~EjSdKOt`8<>g?v`KtP&c z(Z4HCH5yogNiI=1NlD*F+d$pSaFH}NqQ?4}*7>0R52EVrq?9?JNLk~Gr}hbz=jE5F zBN}H8m2-3JM=L=jTYILK2e{q(qNoi)qQPej&U}}{G2_g9wL~v0aDCMaa)y}AYxJ%s z@YvSR7JbX^C>xo47S*HHTiyNriZ^oXDhf|RVnKsa#}@BK?QGE?x#9uQ_vY<4h6K}z z7w7r&d7J{ta!sP}rE94J8~Mq^cDvp%_QAvBykGa_Tf~fLK>661^Mr(^QJz2rytCan(+q=81U3 zIanvvo*_nDJ`(x;0nt`;`Rw}FuU{EHu5D5WdSw`Ooh)8kJGe|ZJyGq_XO#_c3*p`s z&~IujqY7-i*BiXhUewFF9~_X6wpn|f6K!r-t0{)%Tpo@1qj13ptJ_M=r2E9}NQTp% zp*e}a_sH}3xUn|jH0`!@wH=ifl5>3iROw%?56RQWw>IQt2` z*(zbP39n5w%?gcZvcxErpSdXHslT5xZ(Gw}df%t>XT}2q%0u{#?!q{8L=mNEL{d^O z*ue}=L+H;cT_u7|jHheK(>B~bVRyeM;$1^lg9s~w-m{N0mr5-KIqo-Sv)?Q3jz(uX z+O8Sa*47?v)m7zQtm9PMtp+qC3BuM{L_F%mbvmRP_pQh*%<{Z(5Am={Ftyc*pP%5@ zS4oFNw+&`Qon`e8@BH8Syqn}tv~l>I_fp)A!aiy z=F635r}jiun~Aq>-@ct%TI%zNHVU*t`1y7ER1PyvBm&a{u4i+6^(W4x$WLEDUwS}f z6Ak`D823(+d*McY58=_&q_gp2=v&v>)*C=C&0vkCi0r&_NDU1i0#gqyEl@}-zz8xBeYi{)9v%t(?hhg`M(?Czsyka3g#BS2?FP4(Ft$O%bj z4@IGZl7^j3iOwevOzjEaDR7CVJ;=%>FR~qTc$)cih$~}kt!Hs$8aln~yRz-7#rm-6 zoeVl~+a95`g%ITpvB!4nQ`BEPgj-pA*h6=s#_joSfP%%P$B&Q888uCe|G90K77b0d z;9hX`p-^fnn%vu=)g`H`XV(G(1-cFeT_tmVF{!)Q;Y+kjbUC`P#AYzVk%UTMf9oE0 z&lSzd&Vt?JsNFGwq97paS=}d}!g5fqxIph`mD!r##l#hybF$vGX2PYJpnUw;Q?XT0 z(n^9E3XvWg#M08%K%gGq?w};!eizxK9ionqt1=1v6vO^+{#i-WVGTaMS*G2a7;Q?bFheWVhxL#Wu@M)HyX9- zAeO*CIFx(g5a;{UR02*SNpF+}z2<7Uv`>YR*yV+H{OUxxS+7lRZDX8;ECoAzOv=bi z*O66(lj)ivqPLmIVX~col_Dkk?q)rIPzopN!m0&!AFKp*SNE$W`X0{~-``d#H*IOY z^Cl;u;e8aR;AQd0$|I$Yj?Vbb-Q8Wsjh3RWggSrVnznWs1+^b&I|1IWS*4 zS?rLOp`)H`qcSMWAy~VMOm?g1W2kb2)qT0XGJ+>>V2mL`!}X=RJMr=|rMN7w5F9D- ztM~5ijMeMh1JhNm(l*)j^nGRp{`5-^%Kng&`6v>2Tg}61HC)u2_-tFPot<%s(Boru zCFy57zl4yGK#u#n_d~`vo(&MLO_l}k?P0Qqe9NW*7{_lZ8G86elgUZNTtHK#4`tN? zp3-q6pp$%NY|H{1PkW}GpOCLM3Gyij7eP@K4#D_W-Q6!x37Mi{lW;}t!{I}FiZC5>;9~d;8Go&?v-cuV+GSN)qWVsrgJZ_}sPDt#YB9_jn;xzDEy?>o%>}SfkM?Bj{wX0*4yT zk=ca{nLT$yLtv_9XSajpQ;xvs=q4;|?5)+6N6{ZCAWYd1+gx~!>FQ@hBqp|AoIkjJ zq}a*=2a1G*w0>Jp9{er_k)8iyHZi#eH67JsYme(NV`b=JNNk3jib`OHky0*pM+c4H z(WLXL<&pU_!y20P|0avDqZD8f&Q=BNQg^dk4ovNA|u{`}BD7R5jpFDo-zll=-4z^0G z+i({yM@}!M1UTZpDCCWQii+yY+%*`XkwK}M3Q5bhVv}y#C{&b01wjX5!rX5O8m6Xk z;ub2I^eEVz#OcL#2d2IiFm$SPNgi0v;de9%8FzaDz9y&Dgm1N5laA)~+Y0s@(>?uP zgWB7WwQV`c`ebP7DRETzPNk=~r_r`{WQ^&qFZ^{53um zGYPGftT=_H#}5&&f}tOc%}gEo6=33pbQjgt{E5^PI^XardQd&Q>`Q zeraes%>(EIl~znd;|}wAcqGhD9gg#u+Ke67g0r*F435N?dg7GVe;j`^Xni|jzLjip z*=I$-ZrZ%t8?vyVylHjjc-PrG@o?sw+x&{D?=;A;cC2~IH=!1Wu49Gz>zldCr!Hmd zAr!5wJu1g+fF}2T zFg^$0!bPk?OxJjo$-7{D^cdx31fK_>&l?*XS=r7%d<#cSIrMwn8&s~gZ4{0yr5V!WP-8y96$(Mr! z;?-MKAN7JZ(|otrG6RHMZ1?Ln26a~JfAXvLrQwv=o3&^c%`PuX3}kd$4Hj3tQHzMQ zC(LPiuB1c|{w5@a>MqC#9#c@<4f?8&Uf3z|+WN^&@Asf02#<(!e>>_+eRxx!@7&K|Xiu?9ge zxkISvEHR?}xHB zTJ=S-P4;)N(PkRe-R?N{%&b3aA9Q+QjWHi=8;lj}EpWSR67tk0K~W~)?7=|5k@pom ztZZ9+>w|pfv+f>nauqn%rp6?I)2hGE} zAunGvv7V16qWY|zT4HUwCIU|R{sCD?g`TycC2)t?PjX92+Dc(@QEG-ll>y@|9<9 zSC{b%bF9ZSC&{dHpW@?xwr8IZeYkP_lrWh0QCxS=T$cCe&!3T*!24@qaG-_|2OGc@ z?yV0bJ)-ph6~*IdA&%e`ydRou-jBtPZ~ zb&8bWvz6$DK1bTer>6@Wn2Zu}q^6-?n_Zp@m`*kr$bboN_5^rUe&PMvs5i+bDlF!J z>7Ng6jrj5vQs_rP%C0R5d6%xgSL_v(@u3TVAPlPfbl7r!X?m17dBwIN?7T-)WP0 z*qQJTu%Dg%XaA(Lp0?=71rlV0$0lj5G;fBK%h1rAY9v1UPD}UVQf`g)c^N0mQW#~W z2?$2bb{zwg1IhDbDgK-3 z{XcsN(wDy=d^rR(DE^RhQ$4s{_X*7jhRN^+Aa*u(N<)s$9J6DY8qeivXBeRynAKAb z@GN>{*4D12CS68qIedzZm0G;;4|30!Oquy|4bOVtjy>`n9X2*Ytk^+29TGqXK*KwY zeE7N}B zFJ*KASv8nkT8cAI%gTbvl5wqSl|^12&vuA?Z_VlkcF+gG3svc^Fe*hqJE1@O&gRwI z3dl}f&>AyG;auvaadA%5)qks-tQU|vG+8u6M5#MILPwW{9o=MJ?@b+{KRfijb7)Rp zhY9zOors1^y!da(kwE#+vbHqgblei~XWuu-{i0Rq$MvII>U+DYG?cRXn2+4D#-pI* z1XqeuZX~?EwPo+R-+~Tzjcb*ICm{qv#pKi!BS7d1_Vx?Fa-lJiM-YU-`AX=X&1q+IPkZaU=9A1D^fB`lQX_Id~T5 z3VCraJD#d=pxfCgA*AKy6LC)oM^tUDwS{KRuK(^fDUit&B08pv`iBBc?YVvb{sN%4 zF#-#A2kR5pJVPtcOZ44a3*#&h6JQp#@?ock&cTpYp#V&DeAMJ*yF3}}?^eUO;w1Qv zD{~mVBFcNT5?6acJDc7GZpGO}TLBi$grsjFmXxpy-ge z__h(d#VS(pt5ISidvg`P zlOmq%q;lYf#vW}s?`#D5M%Tr(YmZrt-SUqB%x}V?8+Htxj6V@=BvDMsmyheuHzYrb zk_oz2gx?~rkE^X!MQN?$yA#p=^RrRGRV2T1X3#tpEoBZ^{`_V=5`{yQS(pK}04gjW z3o;Fq{|!6m_^rlarYsAQdh77s<0WynO%E<_$mlolxMPJB?~~3M5~k3r^nLG)(-U-W z+evaq(d;*|08Rp(*8Mo_%K0s3?V~9|Ui;4+c}?3pJNmnk_gyc3yF$63u0D+f)&RLk zdn_0V+0e<}Mq8TBW+osL4*ITLA!-vx?(HM&`QHpe)hCZBHj2NWy4?#*jb@JvchOkxWH1Lw_7=w|bcsHoCpLn`x)SRU+VY z`O%Eu@mJYLz!-t`dO$dNeMYq2bh0ue{j&~g-35iKE56(-rSIT2Ybi<>PPTTHh%We7 z3vjy~Dw*KytUr6s5RrQzjnjN~nS<-zrsl}LrN6RRp;T?le2Ol*vwdqI>W!tT>S;JTZnPleGL*lP%RiOmfUx^Cmc*kpQcm4zw(|? zT1s=dxMgyG#$_A*c5^c4);?#DQq4suK4O2{NF3l>fds+R+{5)>1in@XkBuMNMV}Jg zuR1t{8n0RqI^U~6tsw$xTZJAP+4up%e&xvZhr`*yX#sF@I3AZMJG<7$vuPFLG7?uc zN2M!Zs){&_Q2p7>ACQ#P@HdS9i@=A>b;0ob~EQIP?9^9Z`(B!HEVbM?_c*L{i5!Dn)f} z3{Fl?_%JoxspC6$jJ#Tmp_w%%iu8c^OP-g98;XMgy8fsUf2(E?gUnqA1rJG4c*?9V z#|`_Qe7<&z%!Ch@gbP>V*|T+DAD<6pa~}|`yF(-QkO(V`jA}{?x>6o?qSFR*E1$by3jI6gKiQ-``=A>%9Y4*a>*yl>gSH}0eAi`>pAeJ7A|^biIz<LOVS@eO9VXEz7r$0Vh(T zNdZB}r<{U?uU5TVTWN;?1XpgizDsCx1wsT#y2%Rc-X?M1RF?QgT&lq9-ip)>Nl4svo21Q|3M!3i$v>GAk*diOB_XP%(W6Ja?l z3TapG?svnx&2P}R7gHsN3*UBdlXo2nVexsD+U}LIab~+55 z$?U|0@8swl>KATC0Z!rs8N_4;^o*5qJtrq7;l&3qAs^5B8cJSMqm$-d8cg4aL;iZ# zxh?j_(nbP743;B?}??}lH+ zl>O$vV9pA$RYF!!PR9vEi=t`Es4jzpnTmoetOH{n$K|kj&x@W?6*41}*KvXZ?x_1vlbh>vqS$ZbN-5~->siwkv21^iiIK5afS~|+ZLjw= ztIW@npw1%6k`kA4To>u@57Dt_ee?D$T<(qFZ0GHG7;He$qf+YtCHmf`!=r^Z=|(sk z9i|iRK7;eD+IIQoGb=0R)!|Daxc#K0(%XBc@Ph{h^#`x>{PLifv)>%M^Om9@H5DE6 zVZYDk>}ee2bA=otr$__VESQNSN=7ih1O5i|4u*KQQpA$ zWH$?S>;7D}qk|#Vfv=Ch&n@YTX0CwO#Xi4a{e?JyjS-nLv+w&tnGA-XzkVHYw0@`| zAt{@)&u$#@22nho%e9tielu>$7Z8L?gA8cj7m*m&I+rbs@vO#vJ)uQ9Ee-wBj+E=i9Usz2TZU5(|T0yr#_4YUc;H z?jVu+!L@33$ewm~cDQ13H&wB*v1JyLTT#+8GN6wW6Qyk2H?aXW#I+om%%+J6zeswO z?cb^7fNO7TET2{EmtL*lh#MMP{n^_a3R$LUd4uy!L-q~+b=1xanT1x;k8Wqm#2$M_ zfN9UCvI;tC++0JuMKy$ZD?6KK6VVxyMsw_ZQbzrk`pfCb^?yqpW37D-vTM#t%qy42 zB!$>nFxh6o(9keE!-NV7wh!~ z+Y7Ri;3d;;%Qth~yHEupX5rV|wVwU#Ad-H_y3o4i($Iv>$cZI2cLo)i9K!^i}BMtt!-=Vo(?tD z&o)yIR)gWj-4F+0Hs8Efk?IB4v(~aU)ahe&2AL(4Z5km-;^2!4i0(FLGZ_xIVyyF3 zx6m9|vP(8HkkXGY5c1BS%dx;#Se4it^ODUr8z?hDzk2OzPh~@f5y-zG>*+EGT2gqY z7^i|wPTm3&=2Ej)ZnzW{fOuowAya-;`t?{zwtahlbw*YeMnZNIzzC{K%VQ`tO)}sn zddGO3#>IQtmS=KuGQQ59kDtH2OUl7Y_sfcHH%6VhY+Hz0rTx$Qz}i)c;&9YWKMUPZ zw47ppQB}O@$-^oViY!r^FJr&NN!b!k`*T#BTr3oXG!Yb8zghRK1SS~0?#FjvU;|Y4 zEl2CsZkbnJP?VcbTz^YJ_p?3`nhkr{^<(+o`CjGh8ZYe=S2A zzF7r$Yw^xVQl-J7p@D@@W8-rk`<2)17c;`pP{w2k4d=DH5V4UFB@$^y5R4ZVats;M zUB7vA&VIaURKv?Vj>%(M5xJRR5rQ1&DL;Skr%xTi!qCchE^mS-bb_XI->Hpv?<9_z zhUO`C)xisp3imPsp(UeibO)FjwmU|FeTLQ)q_8%e6D~;hnf^><3M%fcb+1OK|IjY8 zo}h2)tyI(0)WN1#lJY}KfVD-zbAka3>p-pYJrrwe+5RFO11LeWv-=rp&z&?ZW(3aV z%L~r}WM>+gaC%eXH@6d`_czh0W}Df${QH&mWq}m6g_pa_|>SaA@f7PIpG91r{Huxtlmj|8o}_P6tfLD zOekM@)_OIBznfA5(JPl9zWf?TSFwv7ZNZ*v$GtAabX6cY3e0o;M7+9E9Dz z0G&x-A)|dpTVQi)cA1h}`Wd(hFn4q%nJDm9oZ1cb*6z%k5K6v;1cn4wnUE0iD`oUF zu(>#`2BZ52^F9ZK1u{xjn=LqPx-dvax;Ft&XT6_nqFi%<4h2MOnoeRWNNGTe<(v@G zl^yel0l>&s(0%Dkp-P;haVK(#eBO8P>=8fMY z{<8j1X6O*21)EK9UfXKww}{zpJ?%<3=LV+E!QpT>aLAC2Mfc}EAlCa{y{Qi#x%o$M zQ!1Bk@q!qpH7z@{OM{t2Fwv{qd||3Wzxe2sF<~G+F<2e?XG|ReyZ-wrF>gndMz_Jq z;Nhr;rbaNe7eMncv*TKUbo5IA9_#Dfaqo-j*Ye+h4wRfMz#1Is4nGGHb!yGMwVJ6+ zj}71H=trA!$w7q;ni!WeTcX?PA&bk>^YZmksntTuF7r%%ca`G3chEWLd!|kOB4DH7 zx?BKgayUV2DfZH0WMdr_mnk|_Ai8s7iucd26xJ&jWwbu1UtYu=h^{RSzw;faw&P-* zPV8`ytk-YdLyz9yc76RO!iWgF2Qc)}%5Ua;nwM{|b3Q;XRqqi>nM{v{oGs(o`*&;4 zH*gjG?5C}*Ws3joI`JKL(N4M!n8AYhSXzpCW;Je5MN2J2+RZNCCue`9`Vn3*#oIb( zq1MaFm`SChu?l1c37mr#M_kO#2`U<4pe{`8gfUd_+avUmN$Jz4&HYd_8%4~_mx$Pa zS;=M^A*cb=-RH>RDW}@YD2a(01+p2@#ihk+o{!EF|6>-Bw>WLN#MaOd1oi?*1!P|x zb0Yg&GjtbTAmTvzg6;yzES&3$zY)OyWT!sF@wkp6KlA{9{4U9+?`Or&VYYN(^V#O( zh?31^0Atl^>?v8S3rf`eQmLR8Z!#|d3_VCDldMz^7Zf=F6NJUZc^4FrFpH&HIvggO zIr5?rkB4?EDD$QPe*`!e6n&`@;%{8t&bciulXl}F$)f-c2y%uwvvIjs(XtSSgYl0a zPI5IX>ojRHfu|Zhq*)YFx!cdc*Z^!CR1V*}O_zqO-asi_x485hgLpbIKHjvr=Tnv( z)(6n=!EC4pkFf87uyhfD;2vSO7=8$BGZ2C8ot@-VRMC)PrH!mE|4PAxu|!xnio^Ye zW2ukuV{QDsGYzgGomoe}u7Enpxh@f^7a#~%a~rHU-0Lsc(9lSERxs@Sh~$U*QryIV z*lEwtem}QK;+a&?P~w>x6@>%F=jWMkxZU}__}-_@oH@^#aDVlR3798w8E#RKD-!Y8 zE;$RAZ^PgF<`G*FX$F%0tn?J(w&K^h!!Jf^|6+zGQzD z?*sW38g{!0#1t+Mdpw?c>m9^r2jn%?!Ry?&~Iqh+I z4`jj6PU(na2UyBaj%!)FyR*;b0N?I@-?G#D(BF;&ZyvzC)EhxiYG2b$>Vg+38ZIcv zTgV|rwnc)T8}+*a#P~J^HgrQly|Ul6iPo8kp zZzKS!#gdK;(3G<8F7;a5aYcPCR4`-04;F4C6qUoeU_CSUopR*`Z!>3kN!Mo(5=41> z#G!sGY0?FQKa$fV+a&JB(deP%;_`u%kJ(_l?b&Z7j+~*_d0A3W_tC5PRMY7Gg)dt@ z0Qe=gtf55ad|)U=wcN}5g`@S_?wTOR`=}_+loTzggoD2fp8E<3(!TJ!j16=Kle7f` z+48sm+=If90wgae*)ocGja+YK`1P3v$0Q#jk#7uy&`RUE3-srC4B=cRfhN=C{H*i1 zxrNBe>h1H}b3Aq&HXuW-yu9#JK3O<4ohrX0i!Ph!zH-C7Rtr}!Pe}#b?3Zz&_pXcT@0W!~%(nIkA1mYonHI$&T4TRN!C8Bkhn4s$P8^vAioE4;9pt3@B zzNYZ*_vaV-B{7rnYbC^Q3Pu#;uC6@qaX7!k`*`ys$3hh+7EXMh;dchJO%_lIrpb+8 znlk9K{AY$6QjvC2(cmUT`<986dBh7PoUi^cF4JVgv$CnD0ME-FFrh8^1gZM=-n6Cl? zxuvXMKLn|2?T9B*hWcDXURet_o(5_ad;n2 zmb1Oxeo5TFQjq4$L~t}+u7!%jrOaDXd>#4E@+(*HFU5jfNBe>QV9H2f`Tzb)FB1~~ zvHuLpLJapo#+j9$-MkVnU*3&fvb0l@0i~NyIV$h!Nt4_yYI^!AO~f1dFANDa5sB_z zAG`MM*-!2QC;Vr)41qWD^(ztLC}2>MurH?Uu37MqF)2Cw*icHmJhWVt8;afhR#MCg z@63m|ASO=q0sQK{cNp*ofW5*8lLK*MUsKn50tcGmqw(0RKQEN>BJa6hoII1mf&fVV z_%Q~soG&|=;S5qE8orT|O+z2+*Iy_{Mu&Y*LLvZ6{nb2l&;BWtQ$va} z$9gyrgWyX7q3X(aHy8m4&Dy3(!o~dlq{Wbnk6#Zc#{$5w_UcGu(9Nh=|;cE%fbew*B z&N1i9+xgAcZ+Bnk-rSu${Tai}IupP4yoWQBX(z%8^|){bGqfA=p&HsFKNBG0ButP| zoOOgDTX?L-S9My^C>aDI)%j0fRlSzS_F%sZXxq_{Z_5-I$mp3H&Wq#XtwO;T4I%br z$kcQZG*Nfiw>vo`kU9!x>TpqfDj*PfCqnd*gOaDX?`^Ng$Uo%bRo018Njyj{KKXhGs!UrZyPEmt9P?d+WKoR*o3i%ZKy=iMJ>iEySEDc34ZwASaMY|x_N z9T(?JQfK78=0z>}nbqDaW|6mS;ZNsQU3?#;`G&{GU-wMNBMJ!#X(1dddGYl%tNWyG zxm!hmTf>GBMSC=l9$J$ieoOe*OWKVs;-DHAJg~4fdbR9s?elx9fO7Dx!0D@r1KLG- zL1hzWJBAR(b@(&)1M4Nbv5Ub|;`&4LAJ6kCuBDM4zDEofj9_^G(3JLhHvKu`1t3A? z(znn^xcy&#)kYki;<7EOna|+86j&~a$r_^3?@hStnUh06;=vZnBd{M3SpOsNi^c;2 zht;AL0lKo6-4kvn$nnZx85Uf>NF-~x^i4?Mj~liS`}!JKc*E)3N1F$$AKG=(rB@GX zcMndvxOuD&CaF10RU~CdE&O;LoIfa5+Z*f*%7Rm5CO`aXUxuB_NgXGKn3zFNJb!!Z z|3%naM^(9Y>%$nJD4-yqfPe@BA|c%%ptN+Sl1g`niiD(egS50rw}5m@=c2p2`Q~!J z`#tCP{d4x%!=c1_)_R`%p7*?B&e>_)Ys4D%Y_aiY{$S5({Knv}{W{OD`o>+3Mdz6J z@(W+1qd&Ilfw%vFg0g9MzMfMdS1w-Qn%7%H!{!8Sw~oWza&=(wZ(?Cp89L``6$HOl zAKW$?+CWYi^oykb;k3?#Tr}XC@~Zo0c-+>gQDcRT;wf0?2c=4jxi!PX!xl4hj3w^1 zsL_&5A&uEv)T;u}z7A#*V#}uo=uUk|V#z^7{r-JDOChuA$MpHJNxe--XsAkw4mOyx znkTAhLn(vm^w=1!esj<$=D@tvL|h-m5DK-DchI57!@`=bG1{6+wbbni;2_e(ijLSq zpDP{+N^)}I5Bd1{I&i4P@z}<_72+45k3>qRdw<%IAl+j7KJ;`qd~r!uR#EoT_z=r& zd>`tz(|%PuSND3iv=Bif65P>1ns8rx#Ip1d{RA8!x|em@JH*P_1A8th?D;n9fdb}l z&YK2~68JrU-d~IQ@&H5F%F3pyKZ!Z>{Zm>sp2Y79UPrz-)#PZ-oGf%)jKq5W}3NOPm&a97$0CX zKAcX;R!bv9O9}q*<1UJowe@zbT{ISh9nOCj`AE+}t{BS_2a1@}(f%PFKC|-0aDBn1 zA^+C+Xz_|g*P+eFz=IL%jCW%PjgKetC8K}3aBe!EqdpMLnz!3E7RBeq9Ku)L5ZIvR`_-r~Bvg zaz_+zrf32C`#*84#*x-%-3Z71@D19iOV*1k$|^hikzr3YBd`)MLY0(Cb0orN8+5wy@hmULK3UI&YJ9cB8h~T<15@ zU!GBfcBD~zdn}`+&R-Z7x@lRuj1-Nxwnb7@OLQ24Wv!aXAhkB6d-JDX+OI!weXQm; z`>HbIVh4+s^ExK*GF_;()-W&R`i&bR!T}FNR2--y^N;4U4Gj&YXPY{YoJQ&e?@6I- z)=VTyF;sViq-lc`@qpuvO3f@O28opC4xVb!((Q*2iI3h5i=uQLC!hh|7&JMzyO!wA z=)qTg%H+`F-4VsgxH-DHm+`7_kXDU}6!hg^SB`7h#nXIm;nDnj*Xp99lborQo4!B1 z6Yl4jJ;be$BlBfGyXG+?BZ-IzRZUc-PtT7xFlExe`}Tem-@di5KZ&4{l2Y*KPHjhB zFLvmNWS6Gc>i|lkXi2YtfOY103`+}(-zH3eKXvK)-m!#PGRXVOp1-CLLWWy`yPt1Z z+2wDEez}Vx7Bao2l_a{z#mRXOTXrkJ5)h|d$-%1%VQ9@1tcQavJ*1)nliO;MNpA$q z?Yj5S>O6S$1mS3{j*Ca5FypCa8^z(s@^(52#ii1eJTV}#$=v(EnwgSNGY~6RNmp~2 z8S2$g{T8tCNj?cCO?5jcpRIa<4{ z0P9}T%*4e(2S|&BH8B*(NpjVT4L2tzK6{~B$Afotz3Fzb%Pk5ar<%!F*tDw(WN@q6 z$|6xgv%_&^zV*QrVY=`qt*RGo(WSd5)sp#u^e8$m?xESFf3ira7V;JwUr|QCg+=jb ziw0(IQsaqZUAjS=$(T26v0nlzJ(V(GL}fgLOXG*Ky}@U0dw0=1l71zpGBWn&jQxl? zJjJA>WA)T+4z$?*gQ~|wFkk)bn+)|8V2>ImsxhF)$Y*cw5IgR0%-BJ><_r?6@+#4F zhq~^_lxD1Jj((jbCX+GR2N!zhJT}W`w=juV=)YpTA1p$B|2|XWcG;~Dg@w1%B+9RS z&F&^+JHUr7mb#tnj;(ko)o)P~1KT7F$|@tL8|{dX9ezKOlat@@w|FnHD3w3ue1U;7 zJxzu*v3=MqF*n#rRYjr371vrG4T!d;yoZZ>^B04&{jObN&EOHz`*5+xD^T|lHG%W# z(_k=m+Hi8gY*h3Y(ByV|Z_l7@ZyVYz;ylE8>4%^De*8N@^W|BJsOhmbWn}BGtK;D& ztg)S`ORb!3n&TJB z=M$O1o|k9lA~a&)@vj%4&?&pPqqX%*TX+`Fa{{Z{iaVhgmX>y}G8J$fhh`!zElZ!r z_=nN03fS4zvCm9TCsiG+tojpi6L51oE70H>gf1s>zr2C9k)rR%mZMhw1tRgm##m#G z_91ThtmWamdmgKcA>Z+QYea&X3W=v;1TE>~^62k8LnEafL z)Q#sZ%B-e&MX?%Y`H6;PdKLCv{6f@}R-YXyxyQ7$QIqD?&7Uph3yKR~r77JPtv?GL z(W}@jys+GR9?w6xsU{c0^7G4fRgH=@5omF$Qm%6i8~g3r z@o}>^;ix$aLGERIY^>jiNgI3GQ+D>!7u;E4G+vklbr2D9_2KCoZhgCR#mDES;mY1N zJo@=F&%#0L4H90NYBeyYnj9Hz8yRyCkBndw5PZ}~7&>q7x;S0T9C$A~F1A%xS*gpW z?h|nC9243a<*xUsQKx*-C%j%^EAN9`rr!L|{Xr%OfHcZkk3-1jq;cn-dFx2y^VHPO zma5P`nHhP>^zv_>CgAYxH?=#>kSn&BQc>xQePPsV3gP4X%i}HIiHc0mbv~AQ!T1@9 zr%wys9TvVYF~JzCMa1Ou&~WW!4pzsG=rrjRj5{*N3_Xo_hTB4cp3!2bwGkZ|JUCU3v%6$~_9I7Ggw702{v$;QHNOE-qRnwpwgT3Ws$Q=VE-idkDT zVIjmM#UhoN8h_f?BwMY9X*B5a(arTn-*y`PF@7XXf>8Xe5bd9Ne?MtBO4;3TGTA3?%aTIwwc?j7* z>3{vT_=!_i!$Y_8{!Q!owpWwOyL=EO3h2j|*5)K(5GIGq9Lm%&H@*gV)`eUPchwX&;gsNE zb^DUIVwIiOyiS==8Ngxppw@3OnBt3*J=eLXt;7>CFF`VfB4!@TGnRVt`jO((8=pL` zl@r`+>{!djymqOHOaI2mRhs|}eB!QR0<8Rthqx}hY-zLR(k$HFn9*%{y z=PnMRq_{X!ib!Dd$OvjmFZ2zq>#h5;w>L%|PP4Y=%NDTe##J@^r9lJ>OgE=Bi;d&H z$l($Xx#AAve-sxNlHA6;Es!R6j{r~LrQRep;SJ{-9r(M~L~OwEoYH5Msa8xFNk9J@ zA&bW(x9YZ&?r?U_{KYr$*T_gX5%x3%#dx&f{M1q6Fx!EckDuQw%23(7xD4Aijqo-_ zxC!y*xdDc~x$5k)G>Tz#vjR>hrW9hyR%uzV%MYj8F0Te-VE=RCiu=RMnOwfTtt=Fo z&3d!+)zL5?HrbDrS{xmERP8TtLlUbC*-fq*pC}W7cpA<{uld?|<})byFzgLEM&*+9 zcIR)&(50M_Wy`X;;wj}!^`U5`%blW1fr{*$s zgIsgnZ()^paho5exOn=Ol~=&ddTktX>k${1FEq^-u zZ2vNUYinZ1rQ!o1{ zJEPqU1;z|j;n_UozTi#~4qWVrzUSbq{HrjI$}hr2c0B3IBYO+0=qF-7U91mDj&-b1 zL)J#L=H|s52TKK^^DlL8C|rg&xz0(0MX_mgs~C6tjZk;~Ud+5h|sBT;nP{;S6hplx49izMy8kM1@wP-092M^85b0b&poPnXyea ztMySu0=TOX#%&fjmz0lFs zwXm_N@2v~YR%XC+7)$(mS6Wo`?m*Vyug1o3>l#B4@va#Z5~TEs!n;dIm@MO{+!~uc zf8dXH^=f`dOpGAs9ch54Ievws!^1M^lBzDr* zN_`Ct6%rJDSZs7Mr|v$>@HF%3b#JW$1qDmX3Bgm%3XDaDxI(G>)Qi(%R^m9*_V@~}5QppQ zh00dkk3X(EkU#c^-(49n_Yur{(_Xr2X=PQwCt}fJ3%3Rt4D9R<&Z-Wnq!PHWUW_o! zi*}cp#`~ryWKpXRdK4~7U2&AJ|I^o(xhj7rO|J2yNA0(-&$cQZu0%`L_Z9NoKkEt( z4@dt{U45;qiypu=viNHo-LgfW31kmIu$~<`7#BR-H$o_63x!Q#7mm3P3rco$-r!r@ zeS9WNO+Y}<|6S|0Ut`BY?rPaK>9EtVPJzYp4ysX* zl>eIq{RL2qzW?Rvkl)|Zz^-il4{rzg@HiRr?}B{ze)E}{>i?py1nUqAvdDHnZVcr0 ze*9Dcq{3Ic>Zj-P_np>Tcj*+oVd1OR=0JrXNJ_;v{+9~;&n5m%rjfsXH{sc1g2(^4 zBz(W#`qjVJ`Fr9x(G|NHsMAD0Z`{r$votbk=+uj97PmcW*Jtfc)~W3H%Qd=IClD0Rmk7zo1f@$tmqs7B%7KsrWuk%5j9#eRfo{uEu^vX&^zGO%%h>7H4mvV-n=XaxHV(K&pA@H0u zC|j4dbPS=}Ee^nOOFMx~g-ON7y~&QxY7hmvqWcozG_QlqsuR?hQpb$Vx0-u}JSl}B z1s8`ea@cRMK)rz5CT1!mC>=4(^GGsz01k8EpoiUxm+cTM|D5L96L30Fzc20c3H@IL5+oR_w()D zLq`&&8xfaGw>&>}FVdzi5}AWl3vu7x;lDe8XLu@x%YeOAc>CtFE`rYIecc}e$V%?; zlnR`CrDh;XCnqC%~BS_R(%sabl&p!We64}w3>wie(7ce=BpvV1Y!h973(@w7d z<1&1UeISu@3cl0ueBg##-fSvYY9^Uouu$+RY32UGK1`HYr3@2sq>c8Eh;_4Zbd+2Xlt{70K2wzlpL z=4!vvk-IwI=2)s1&4*Hyo=2NDjCXRdw;$%rcurl*HgHzUZW`; zDV6^RvNzmYsADDO3tPe0P(Tv+v%6mk>>aVVXGl-aqY{_Ok2!0!3ms=6ZQ>M$bmDw+NEW^wP^MC8L@t;eCP)s_c zunYW6OK6ISHQ|l%YzYyvng{}bsJY*@z`e0D)8-6YO16O#?xNeoCG+XQCTwO#yN;K* zx9})Kkz)dJkHxAjZ(D?JXVz;VOyhtHE7TuD zV;LFoC*_Wo7RD@SN>!jyEjQuat%-Z3F-`}v4wN#GDdehMdu@jgP`F+d(KonxwA)b_ z$9~K6&uG`*MDAnNp_F?+4NP?#*9qL}Rm@RA_Yg3T-_d0uL`N@|@>?(!hogt~8_hql zo=3)6`kefUAPVl!s*_icW7D0Tx!kr1fv_c)7iSopCp%er)!L9@Dk#PK0jt3SphI-e zhM$Ut>?y2Hc0#P~UZj6PDdtani5{8}n4;Q;52fF@0hF=5y8Mr+IllTub}_H?1wrpU z8<))6V587K$@${Kb)gHCb~K|a$&TP)$ky(zesh=U(mv`~v1VoD(w}2I%Gbp0-x(XA zZy6|7r7ZgcHXmGB$>ERaOKJcpdpy8{;#(CclO?PV>wj}L?(+Ok|`ELo3Isqy-0PW*Y5`WLt9>vyWh4Jkl>V?ikF%VyN ziAnifFwFAu{9~VgQnjYsbqZn!CDm`=A++f=Cm{HgCKpi)(Yu?~+`pW1E>B63>S2@A z(PxN&v-664h+@23PbfpbFwS~eFYJABX^x=N{+-ltT26xd>+_csP)ZsG1_B@|QaqkP zOKHN;J-wq&z-o0Bc&%4P&6ixxH*P?Nvw9AESz7MJU3Aj zkEGH3f2m=oGKF6$L-rOHFR}-4sfXUN41Xu(`FVY30byv6ZoM;S3FWNFcMio9s9gTi z6&n+vEc%N?Tb&iH-T*QmOiengC73p@e`>@yN# zUPsev7->ay=03eTBrPsJ*D(_6DFyQyvq4N59vy~Bj7UWdYBjBc?{VE{(0s(Ufe%-V z<8;0y7Lv1WLH;-nKt`7GvnY(^%ZnS=(N5GeB!K6Z$c`|5Kk$@B`>K=4rQ7Y+ zRxyD4)?4!rEj;S3df{aP3@mhMoI?X4~7G6Ahf zp2tW&!0>WO(M?VvM@L7ew;2^yQ=_?kzdJ=6A%H=`;ZMw6OHM%nW2!-D>&)4%sHjL% z%5LfG6iAW@36F0qS&vnzG((js>cyy8?7oYSF9p;CF%u?NM12pX+4ME!}#L`5Xj;{Ij5<@RqEPU@q5+@UBKE&Y65MeczL2P~0-SYce0t8!meeaWg&7Wc_&9?< z0W`qENt37y)oPJqhDTVnYsX(<wnx}X|wcS3ciB#EI{ElKa zM$bjfdvpt;b_ZA=U{GK52HM%#?QI@j(RShNQ&TE6yLPfqeEkJzAwbNRl}EId();!9 z*~cnnsh|RV#pV*~E<5t|>uQUhV~W4k#_0YNcJ}+n@jG&9;%`}^Ma9HQNZ2oR34vg=mX|$q1LzQ@6XE$hEEHueLq`mSZVa0zF%kgBbQ~8lu+Q<_YijE#&ySGG1TC?qx@nboM)G9eh+UlX!}+o42s2+RX7g(uz|`#w2tla~wK zl$~wuua51FihL=P**`ZoXUn5VgU1w2i}<=-S&gv23nIyL+otQVq~7;d;cXrgpUE7a zw}0~7rIt^po#$!m&0ZX{aVTI$>W9{Qt+m0qpkAxxMt}gW(heZ^VEFb2x91KVW_f0k zF(*%BOfbPhnEXBzKgjkq6%|#Rp_U@opILIK- zv0j{CPZXi4DA1naY5@yfH#wZ0o#lQHR{Vp>>I-#reN2P#U$|;0=V>KFOaVRnwn)%C zUGmDhW&*HcJAE6ZAF2Ys)xG)E-WNMo+9&f=k%OCCJrrx2UGV)_`ZdDWY=GBALtetM z)?>sPq!F<{{pIPo(u_*T>i5` zjTXj2%gyrQU+n!E zXKtaB8s@Bae8--3C>$Dl$1tx!30P>`0OT7KHrr>%Gl>80Tw^(e?zaV&D<=xR{qL>YNvxF#dbE}O7JH% z%`dOmT@=NmYP-E&ALUD>pz8f@DeuO;_e?DpKeSa6h)dd|9pn0jhVQb~lm>-gngO*M zudT0bS1z5Z3_bO)z0^P1a|=4zCz@GNyHzY6Z|)sV#9oigqza0Qr(^fCNgS$Se6MVw zfhn~66gN;pb9FkL4;Vt|tdKtc=LE_W$$-OTElawoTcze1Nqt7GEgf&# zej~M0nZm)_o_Xs#&y5Ch0>H#^_S5|ETK7byIs0t0R8mnTH}Gtb3MTh0xQc`Kc;ts8 z-MwaM@~snAz^H&a-|olgAE%p6w5R8;Mm@Kqfv5EWI`=gA#kaWlm|EwfYRt!%A`Ahc zm>k>j){zZyI&V-|8(iF}^{WY^QH-%7N$`)|LMQBOctUXL2Y3yPGK{*-k}!K#3Cvtb z8q!Dh&Y``OSlHLQ-Qc2!*%)YU_gynJGH-=Y6qMdf+;;aEuWQwhplE%LZ@Dq%%0Q|7 zsMXhC zpyd&yE#LqXF&U>bHr#NMc?5cnp7BrEV5ti>2=*AU zfXOJB)XuFPHtb6gk%e5&wJHh)Vs|2RunGwduS;sU z0gJsGdRs|Sl1I{Ozes7N1UYQN*$xLD_te)_2r_D8{A~2*d%mAP(~F3__;!h?)D(5} z3%e-JdwINrS59KH{?XX+>+3JCM> ziU>iW=sX}T`5F-+uAuOlbr5uHoO}0t{C!cx9+NR9`1n{DSE|rD8k_!ZYLdy6(+47% z43-H7+O@hQ0ghpAP?;1Sb>-yHo&g^k8tb8yJ>9c;WB7R19_}Wb0aVf?uTloUzg3rz z=uSa>#3d`sz9Fd@>=P7J?56c3wW?cG=7a5u2Zh+U|CbV}%PiRxw#a-93NEglkERc- zS1Eyc21pYc6H*@m<$1ZSC43?_HWWxc`nB*MIr)(SYohxMm#m&*GfHc(b#%&_2*(CzriL2`C0**_m=x_ZuS)}gmo_(wSuNMGTFJklPAZgGGnM!^>b^N^l3rUN@|I>)W zALi$O|EBeSv9kXMWx7U5{ufqn#Q}j;uAw#l_U}K#8?RD^Zf9z0{~Oj5tnvL&Lt7R8 z`F?$^@W0=HT_$1a{g+|ABrGuahspg9;~H+LqbC|pGk~P&p|(Qf&aEF2&%(tCS&e+1 zmzh96wmNQm1%SzyQSP4)_V-Wm3g8Q6GyC^kX6b;v@Kws5KT34}6|d(bmo(=gKq^~o zWQ%ie*3zCqBDAUV*y&i%zbT`%#rN-f_|KI~;3eMwDI?IVw3%ICXZH)ZoDfi01H~3hzRhk#G08H=!VqcZCB?h z3#0x0_a18^fi*~v9R|Di0$wM{Of5}`F7Vbcz>fKx+{T7BuH*N(9Pis|RK*2y^ zW5=Xx;P`s3{j&vvQR1HUhc8gwP> z^=~4+36K*Gn%6^eHohAd$z3bHSvskInWR}t)L6COEbKY@lW@nCm8qi!Mp`}V{F!E~T4+Z=B}Lrap3URx zRNEZG+TJtIQoD3@!5N$DT!JNeZwK9~247}Zi$_qUG?Fh=(m zT=2?AWz{kVElY3Iuqizs@m#@_Rs&$)gJWW@{%Q?vMiTI0;&}Le1k0p_E)-%F<`H6) zp=6cMJm|#!$hYIP<5}VQ;*3>YzK5eXQiFc9w#=u5fe(2OOHv9Y$?$BmCtt#^r}eN` zbbsom41l2OEg8aSvP9*=_Lj?o&tWl>SK7=BD>U8Tl_rG{=^Q;>rnq`FZ)YU67uh_Q z-KedsKweZrV*Yol?u$t*UjXr7lI>6{HJZ;tZ4>|0D6z13S}2~ju$n(;ad8&yK7X;1 z*1jdU&UMJU*g|?^yK+ul=N=gfw1DV5Gz9hKa)DvKXZrjfKK-S*5mRWo3$kVq1q%y{ z#iZv!`O)E45;g8EN~x%(-q~7cMy(#)KIg(`Pj>AF#@qFZ@J8ZEUC<-m?zN!jXq@8# z`wWIFJbCbU3u7^k{YG>a&OEKFPK@rr9c*oDdl^FEw7({_7^n&9;M6H;iPikd6Jug9 z_@Pzte+EMKt}Am0a35cTgJq{iF<00P`h6^EPgZV26>t=rK2|Nsk*ARN;jZ0gjY6JA zI4Dkl)cKaItyK$*&|y2h&@%i1!&9qvtn6`cW>jk5W9o`X>#cp3zBQiVRxM0DcMIf^ z@srB+w05^R4$Cl@jv3EvC=D*_!=>KY*CzdJCKE-W<-6&9MPj>G^+!fjC1aVq8SZyH zSx~b1=p4?V9Sq`ReM%0M@)<+H{@DII7DAv0vLnpjw0NKAEYhD@5g58k7dUrILEjM( z;P1CjNo#xIJ%aK&n5T|xJ_n3vh&Ub%lC>%^(Ug`lllpk7lq(kiCJ;`?6^A7L=9*1R z@OUP%F98lR{@sX}$xgd$=xdMtcqC={9x9kEObb98P+8>iPlzTO6P3r%ij4 zMyOu}(mRIT*i+54VN@}efZq}qXv~FpzH<)O1qu-F40ehvSNOrsB6--lZ$Y49w}xlmht|`qfZgD6L6%P-pCSEY zFqY#jS6$sl$l=+MI+J3y0+ROPt*HG%$?cccjt?&C2W)%y#w8-B4^zBHs+~oW&hIAP zUy+-v6s+mq^lLwW@g*h` zzvjL$Bi}ky(;$glEG);Ad^Xc#FTiFfAV;brMa;T4Qw}1j+U@ug#zs7(vaIhchRLKYH;#x@9j`>ap9<|tAkCyZ6ndWYLOKat)Z4J@ik>@OE2D; zx1hrbq*?hIl^VSljW0nL$2Qe=xgaFi(T-Nv3<;oC|0vz2@PeM#MR-WB;0Mz(} zJ{wIW|Fv?&6v?1X!OeZUT%&H7@g8Im4R^RpWlQ{TFsFCwHsyS|e*GjxfhL+&mk)fd zF154jfW9DDuST)V%d^DLuG&6R&TE5HrNACx_Q@lkJhI)=(zZd^^xB8qc;Wye)>jaL z5&4zdP^nUsX??sjD*#8{q|+|(8L4M2k4fZ)NOCfaDY#|#l}6$F@9~k_ z92OP~RMNzYX;|?Dm&>>Y?tFM+Ar`i_?e;Z(e$Pru>`=Zd8i8mB4Y)l$ZXC*`8QG@4 zjurxb4Nw!#-6`qB(n<2CoWZ=FXEo>ILuPBEg|i9dfOpp&3p7Qrh&g`+Rn{P3qe_hl zF}vAk>$9Ux(293qy_bQT5tycxA@i27FNGu9vuwZ^!Zx0SGLY7p{Dobh{q*zNnuO0N>|n_o*(l6LR=z2 zF`m!B$cU#ZbInxCX07GSZ&~PP_v_420{KMrLOy5v5m_B0QaXu=<8#*E85>K0al5s( zbvm-T@?HNAbie!bi%kK~fpbCffaV8IvEZ25qSM3e1XGI@SgXNDa?2DBU*p~5I8Pvk zNdZLeElu8D&f824lGbbrOlYYSI)$J-x4%(sVQD$*O^irZ$oj65$pqH44lvhF$!CzH zi1-`+UP=O_S!6blaAot(Tf{{o{8B2gZMR*txN7=6ft`gZ(+|O-4rMyxX}9?KwZXZx zS#!>pJw#8=!ty{YWU23zS8JumGtvD5{f5X5O6i!!Dh30KflI5T?n53cD?3o@=3~{u zf}_94WdfLsSX;|E?WZ4!i;LS$SkGfkgj8;uc2N-ojojaqLV!N`*M^&JcJic*Urr(0 zJh+{-VuC0kqIE>ihY|Buxy`-NlDGc?n5&T zo8kE5P#7sb9!B< zqX0k8s5`poQmgwcDX9(x%wj^TN~YlGZ+i25XMe{Nl$t+W1J(AMhccN8<32Yt#KVpH zv$h3mq>6<@J5K_PTRLbtZc#HcdtA~*wU3WnlC#mH1^t|lVl)Q(2|8r;#zf%U0ezhy zrLJXzmm6f2CsP&t6I!e9Upw%$1Q5M-DdaPE5&|z&4BPuIQjt5>XTM1Y4~+UMT;c7j z<)%Y4adF5X)@LFpDJiiZ?M1f8mT%4jb)^0zAo~i3+sqxiONXA(jT8~7sYPwO0&`&K zb}`PUbiwh9+CoH7>oTtcx+ka8+D&J>b1z7f)N;M1NdxD+%jkP8|f96P+N{DA@ zLR@nEh8n#ORjfi6AyHv3MBm*-OSuB9A9CR31p9b3i|2?QOkCy}c=F6Q3sy1;lCn@vl;~V3W)#KI z&R2Q)E1ryvlarHH>*NWz!<)fU_kD@lZFqY^Uj1DCA24-vD4DoT0OoZCalCl$+WMU3 z-6$qkp7QeYI6mu7Boj`uDNb}iW0H`N4B9%c3>k|m*re`r!AVqbPxuWB;f4+Ebi>;e zKsRgEean^iJ+#ahgb+#9K_1yFSa+k z;`tl=h`F|p*F2F08z&v;a4^cxvD$(f>Fx=imvL!7b)40cf}H$XxOjMeAcfeT<0|i` zPgirye?TZx*{hv*m-0U8YKfbgdI5B3gW+rj zm>)g^YYY_f`9h2uEq=#g^i@)9g03&2%@25Wu7X}yxFOEDTu6XiOdoO)W>dd2b$?U+ zE_hexTV=;{P8G|R-)O=%yEf;7V7vZ&lGSK2zgwnB6FtV{HCtC#9_k7v$sQSao4)nx zU7MO$Ojwp*OjgQ-@U6d5^E6Wa0ojLTj1U&Z35?l)dmL9kP$1bTG=PU0b^_&oZowyiQs?c+=95k#B^ja?Bilf zcX@<*mLH7n{zOQwPY!l190ToKa?@{BY4P=Tl8c3p0%Wq3GPS-{=^Xn)q?a~%{b6m5 z)}Sj!vbtg)CYIxoViL6$f5^_!sK)(}(R-t~)c%{mc@wRAK}(EHYe&bM;jN3q+{5m~ zUc}BP6PP-WL?nwrV_*EM1i)!jH9lY595@6gB2+0bWXcb3u!Dm!$t?Z^`aZ;fTnER8 zKjeyPDeiy7Bkhn;miFA~V=*lyWz1XBd2g*&BXSGQ)9h`;ur4z_D|}Wm?fG9+n>AwX(w7z&{Da+A1>}Ua@Jw4n$tJ` zNQ^1Wo-Ql9q<}2@vkRKI&A(+3)CBmAB0uSmdrR#)9sIFsHS&Jcye=s&28Sh$KLL9j z^eBCFd3O!UivY2(prCM>?pK5l%wf|id^4Fzomu6lu$G9$I@ei3|3R6 zt|)`pn4U;={aK{xH}QF+moL4pIo_@*H9p0`4rXtm(-Zh3vaOU~T2Nf5J7(|br{Dy( z3Sa%zk%I{XIQ(R&!@wAif#y2dA1auR#+y@Oss$yO$6utK!PkQ+5_o11KPj7@BXV(% zB&k~~-|QMtOl@(FebFp$xgY_fwGn>})hRz@$A0W)tZVGoN`f0In%|s}ABKz?i8E0a z65MbW2@$>itEuT=+T++8grpE?a)Nk;I8C^Z?dTyK5VO-<(13C*tP1>4@_><#Z(vag z(Hz~*`M1yN!UkM`mB4|1?}0<^uVPZqx^Gl)01!W*+VS%Xf_fhqQsS## zx-+P_bCCB!M^2}vm@VPe_IBxx@Xhk@pMilO@zqs)GqU`j65@|mbOs?9my?3Urej6T zZ=SyakGuDMZfpWe4%pN*QTd6KKQ3qdy#1E4&HmH>PC+FV6X?gRwz%BpR4zF#SgDjo5(EE6HfU2x?_{`k607Jv_)=5yZbnZJ#9^eLd{yc~en4 z7;5qwAF2mordWy$_1D%Nu0N$Zx?6iY$r2gLImVXbL9yyjn*yVM1L1>y>((O{+NdC} z0KIKxZG07fHjX2%P^zx$JA?Je{IJvNV_ z^dK4Zy}c8DXM18w#Sw4IGd@Ed?o3QUt~6W)1>EqVCZ(6J6a;1NzFHHOE#NM=eLGl2 zrta$UhJpR^dR)R`kmtv@k#ZJx@tcbkRrH4)(bfsOV=z z>3)+sL*}8a>%3j1qJlyy$$c#=m7xObMXXoCFPGgXx(CJ`3&X?1lQR3#mlm#g>~{_` z09|1#xJVyU`^KzCCecD!LE*c3Q(*6Fc8s1Qbi`9GE;Xveenp100g3MwqdzRG@#zxs ziB2$jinQJ3#N!oM&Bhfrt4^K1$5JkD?%P~PK|u+;6*-m*{Qx!XI|C{Tx=}Q3rqk4Z z+7xWKPGLba4!HhuaPvm^+n9>Ind~CBbHZxo6U=h^Twx8$-*T&xCHZVcK6w`;8lyZLZszn>(90?A*2k%gK3ZfAXtmX{o77?^ah>pY8WE2W7gR z&h}>;nmfAkVUmnW!z3iD@;Ji%@q0|Kg?RHTq){_ne_xW2rR{7%QzLd2cEW$yY9?6WuWC8jpr%o z@15P2&eHXswVhyP|DIE%5ENJ@O%fzxRw@B?4+70&r<-q^wB1*-^ZEMlA~kcf;quIe zQp#ppc`lVC-QC@{V`du{cb_saC~s(zy*D!poIIVPi$Aa3Rjk#zcI?7eeONbkGFN!V z_M)8EruCKbyJO$9ljTKHET{PCg}!dS9#Z#MU^il3=VKjX0p5>l;IT-#n3Ile6MHGG26i!;kz`)^bt)h#|r@WdSt^i#fca-mJ zH-+E_J~Zrgmg?Hf<5k;qYu*vtSw${gZ_Uumt0-A~^>^VXrk=xoS!oEGJRg(Ty^12} zhpuo1zKrrhg$(2GUr^q9K1c346qL_+e}4l7<*riH3p`MbI-=8`W)I-67{HgN4o@K{ z1deW_Bd=Q@*S#3pK96C`jDm7B^js3{|NjDi-`)Rp6&1Dn@Oz5FI)cdnmI;nV*6oa& zyVgg4K9O*%y(A?bGb+wnFCZ>*X_Vu9s4oyi;FYA58#oY1JXU1A80c=3a4sg5l>}oc zTaAio*cp^T%-wO~68eY>!|F?q0lYYfUpyr7H1+fI0 z+%_64@&v;mt#^a$v}N`{7K{-xYG~T0p>E+%Uc_Sedi&kXiFY7RQ$#CYnt*^PEH;v8 z#ef(){ryT5S^$j8qK#}%3m_WG{{H=We`m}+g`yzW#=H*At5=X6F31igCsawTED6^EGuv?9ij%BWmc5cn zn~8npmRU>E@LgCpggiyHZQ>F^M+LgNvn)r0+$?5i>?bFy40GE~{Wou5N%rYDq7 zTA%yhEzNT(7$dh?{NcRcnz3no_PKxwQ~nTf7=<>;^Rm14$mVhc$x7FxT6dSr@Skz;^>>mZ@4&CR=)s$+>OVooM(`~dmocB(V& z?__t=kn75O>e}-b$9j!BWstb-k>qL>%H>z>1=d`gSO*YY2ApUhr0HTy2jd)>ZKqKq zTNd}1JtGqbXBuxm5yL*XsoA&cWV`8>P+~ff(@R>F#%)%q&VEptUhl|d&n1iHEu`%~ z(l@KO9}|AEUYfaLccL1@qo3@wU!`?l+a{V8Kf8v6S>TYLMT5$YPZhpY4w~>Ab&$S< zd5=GBI9rslnW)^iA!&Ht4;=^hE4K{>vJ^#NANR(Z*Cwy6#ryjB^c#}8XSe6^F#NdW zHh0U>cE5a;=yve2B7RacQqwBUErCn*zLrJi8e8n7%PAhLyQK&Lp@DC1LUbA|+W9pX zf@zxLv8<+fWy2huN2@{N?m_3L6Oa8GAD52h7Itx{bK98=F?RvV&x*qh?}73@v7aimCrjrHwyUP+ zHiCoo0oow#)Gs`r4cfwNfr6=-AhDyr@axogxVfum~(*XnVDa#l{1@{qOmg ztM5$a*{Dc(A2dH@WE6v>C9fzDqT26!Qc4IS^N9Cqd($GDN)DKDJ>`CRn-j|0mgq4@ z4fZFnoFLDznL(#Ap87y`vJ~1|u@OTA9i)Y8FC*M8b&b9)L~6Kr(w85JwcIWYC7^$)x3H)OUrTFwEAd=SzJs^5yC-Or@_#Tl-eE`u#xqY|NK#G zXS#J4pLV3?A9+Y05?!~G+)hois`$*)hVBRc>06;;qg0ZJ2&*kUI^&~L;ZWwyuTo-* zeQkEwi~ivf6IjgPvDw}j)@%wfsSJ6)NLrcFzV3hG%YM(&UXnO}O{Csg!13WZpJ~Er z^x%by1Hy{XE2%$>MbmCsG3I1d8bVn_r$OID18G?M`XAjO{%_EOW+`xNQ}M09#|65m z6dxZ^-kqLMNS$y4j#us#oAjzl2f4@|@^fg6a5{TM7nzO;uuoVNCb+t#g58ZKyLhH~ zrYRuS=pe3kZ|^&{(O8Mc2EoNwv!yW!sSKg&yp3FFEU|igKQW|veK5ckZS9bYi_~52 z(aVvsKl*+)YD3I=r6NnG+lcQ)IWIM(*V7FXAflGS4D0nV8WTU?z(05RrM}w)dcnY7 zqoW_EdkgCMFoRurF@8+waB}~L*M#jV*#I|hb)E@ZY=W~}Djb-}V_q0TO#!Q{3+>W# zyT_~$Py1&-N$f#NrR1>Td0!(rh~HIPU;lJ-(0w%1aB1CMdq|g=1QwQdj-g0(GN-lS z%D_;9vWZEY*EpA?@nGH{bafwCKDuXnYHE-CFbXi2(Mj7m)C;p(1cruxog_Z@8Z=kQ zrl+zUVD9G2o0-`mBaoHtZEI~G43*~3nh6Q{W_`&y%f4T>C8ufEs(5nLcW7hVJ$bU- zj2@0fm#LT?xKXj|R=s`jtgW+E>9DY?w=hMkm(IhSE0wF{{e)EEVn;vnmF`I`B{!2e zio@dW9owu*Z}Iv6+WYRHCb#fS?zI=J7YibY0t!e`K?DRu6b!v2L69OUV1j^jsZmrA zQP9u?1VkVt1PCo4AjB&w0#ZW=ML?y59_b|n_6_dt?9BfD*!j)u?(EDubG@SZzI-`3 z=RME+JkNXHEBuvSuRuM_u_YLiet)MMsp>&JSK8Fv8QV|!)y2%;cJrf2%@ij<03$(z z%9?7Y42Qm74s;bF`t0n8{Oi{ZYi~@WZf;Iaj~`3GxmRHQ_*t*ce+C8zuorr-Df|lC z6hl=ugT+TU{JtDH=tCIAWJOh#9zQ7ya}$&B z^x3nlHoGe@^Z_=M*9~eO4RGux(L1M8^bSM#ktkrXbHuOdemJ?!FU6`(;#TbLvFaoH z<7~YXtv`SM>^B_a*bgXho})NiJ!x6)LYvH%fI_+) zZ3&4la_JVNV!F|ilB5Un>4-jppu%0EI(JN~{FVEtzGI%{S~^6X65L^rPeOa-{09>D{*cPc|JNzD{N5fx7!9?NSKep zdP&f3_K;GpdE_=(#%tv>imod{h?Y-beXW(&)ms~#THcX>0MFfTBM24cS5Icyt{#Xl zw~>pt7;sKG;}c6Tl-|h)Yr;_F=dI_Q6{j4Fa!dRSZGx4h#px@${?p1C3dAEL1B_uN z#WJvTUi^GoN8yEshOxr~UT!K6M=VV(-)`AkP0GuAbMX))QZxO;Y6bUCg;G*d(U^p# zndzM{*vJwWwae|t#1X26K16MIe&6%LIo5@`@YC(;h;hKT53|u-q71kCgqR1e72CqM zdiE%*DR_@M#dc_~bFI)cUHlEFJ(L05P^TFsDme~6 zqnD3T;&IrPoxPhHyxQU&H;t&jS-l=O`B^7{Dt%`mXt_ktbK04=tk$s>7oz6--OG!x zjIFcprv#x}0~u=8m~$g((ED0-7ZMzbR7~I<<3l*^xMO{i(ltEtSgFHxe>=NWI6z$r z5=8*8QnIq9Fbm6GduFW#yOT0I*Nrlq-?(J1M#|rk+eeRH#?K`j>10gco)X=!>bs*W-L zb2|(J7z}NG@(3x z;mB)2xJ43(^U+HYc$xye@fB;(K{&$$k#lWe1-)noxn{?d}`Ps!kE_{&RZXGntfV~=V zz2zGZ>7V*#0g(O*-i3DajaW?~vsM{)A@vmVhKfplAC;tnuDX;$$(v~ns`D551@9&P zy3q&EiZnJpGR&tG8bk}&bRZ}nc#fcz0Uc|v0Y$%|m;U|kF+0r;sw~JV-f6vxpJ^zk zIMo_O6+!>qim8^a+OOZ!!=k=mex8fS@?7>UbwCZ?E-U?I#7;Be`H<9;gSayuiUN*#^p}Uv3C?}Jzx9kH=&BW;_-)) zj^1P{1%_6!r?YO47F78e!nrip7rT0cQnJEj-|N>W6gSnPq-Tb-jIaPV1sD6PV54qs z60KU-=uv82U>U6OYr@}5$#FuTM@2EMJ+!VbeE8{J-u?7xL@eFwx36(*&ebfSg0k`| z-1-QxJkQ_g{-GuunH)6erg*3QicA8v)t6=ShF+GOTCEdHdFQTBc`G73+?W^+0ISH&fkk{ z%-Lo-aa}T;+!q+zpcve!deFzK&b93a+9#%cp-nRxxaG0=WdKMacIzu^H1>GordNA5 zxo&uDlVPtDg#R^?on91h@(SQXs*27F^AK`uw?>o7>B=v;L25gmKlSj<#=R54B++l&&sZ z^>^y6e7FLru0HvUUWh2w#p{l7=B|`RxvKN;Jz1@0hUQR-=s6!)G8&(0)Aqfzu*jLy zwv}V;<|`*IE)mmMmv5JZrr3~7j4INLS{25>^+-lmr}g(IbmreIheh%0*UHMLkL8Db ztQ=um7xGl@Mv>7&>3fYBjWm5IDBU(w`7#|i(1HG2b$zo~puoPe=j&jF$jYkkIyqrv zXSc>bet9nFG@Sc_SkR-t>DN5R{_E!Dgx7|dKz0sHFDV3mVdHD#@`|sn4BjnYCCXeo zTc==IA)I zAe>Qz@L*(FsLHQ*faf0q`k4MiM+ejVYY~nwdi?h7+oHS8@89&X`um6ajC$xzqCVIK zwS%3Nfn9b~c6Ci3gH5;wyty;5c!zOTEbc_2?F&j{<-!#oew=amz6LV{hLivq+Q+vS z17~W6s#IL7ZVC>!U7Kiir;dWp zem6c9TIStN7F4>E{b&K_y*DK~I$KY+)U;7;wO>QU(0;Hnh1|pJAwtt#*oApKX*}UF z!6N8bgvuqfkZ99ITH|P{I{&>&3~TVKwR+>b8CBpvNQ?ngLX0ela)%{SPQlzHY8h(ExhUoPd#1t4M}wpo2qd zFL$LJ3CC>olY^!ZR;QXzExFY`dg#rfSQJ8fv6YIQ-HI?p-in5Wm($_lu(7N$laP*a zoZFFOU2X|&QOtrgD*=5=svf^45_sBDon~7XD;rzgXJTTP`d%Ff$=tNk>A@2a{~_^8 zPP!HDDSgR{h z#mltl#%I%tsZI~~b3#*#wQ*`Wpgv)lhRvV^6#3O-L6f=+tZB@v=#wc-^5^nc1hwOi zt{$1ZVH;~B`UzCf5`^sS0XT(;k$k83J$)X==$DKN9z>D!H6h7vNFt$gI#0+{^#;XTF=KwmlRFJ!p2)O5I*mKm4i#M9^rBK||40!;`5}gI~ zQRnLzQgL;$`NfOD5~~V`vx(LogUcDGb>bs1O;^Zhit(3R(Q{}rYIxX-dT;V?d)8QV z-2l?TA%vGScpYlPa^`YiLyAA^^8F1oIG2==K-2V-vU^uweTm!5f%;|Y)~#E4+#_JE zz~9ZPmL)C*#KhUiMQ#uTULArR5eQ*G8m+4Aid;C8triL!wKRnnm<};=&Lu*f#sO-&ceCJNDi4WV>(?XxxorYBKgPjT-tH4O* zx-Bini3o_K5J4qb$_stZ$9FMwZTCN?ZyPup4?QXSi{g7E)%xcVBCb z+U66INTCI6p-=Ap5>h6~cc1pa>-m~g@E^(#gFH6#{`RWPUVHiYHW&CWPNhBIz0;9z z2SY@|KLh2vCPe7jP*}vnDS!S2hRpZ84tw2k5EA%3Le_46Kkp)sSBGC8x7ux9`@?4G zwfq+DmA&hKAomRW<+9Me= zP~q~QuvE-%86Tp3cw1R+qrijlB-JEhoouP@?w7kyZ`ay;JdjdjCJ~aCv3ALuEb>0P zuZZKx*&sXA3h&nbI z?kd4fytmMdWDrfvT;j}NQ9p22^_eh+z_>c@ur}EDvFW;@qP&F!Br}O!jXRdoxPRtV zw?kC*=*vc$V>yHqef@8=Zv8(W|2eXQ_wh%vr2U3DTY9nMWr_km^3S#=E>GzNtr-wkWi2t{j$^qawRm`fB@m|dHWr2f&Kg0 z^Tge|cSGX-AVmfp{$7}EQW{Jg8cK7w5)z{B%pnqQU)E8**!?W;(zsS=XJ`Z_p};by zz>++qldx-gk?XD^G&OC%n~zU3tN-{WxZ)s`4%?7kT>Yg}ABtOB6@Vzb^8CQ!o2h9J6dFx{&;$GU z-}Olv*SzSX(jZzE*d+AEI zo&4JC-Yk*M+C6DxnPD)9rnA6rXGU!qUHg|0spJB^(LbwAi2wh{BKtqQrvLxj{YO#s zs#f>?b*BsW|LdC6m%VfP=~sD^+;s(-jDm%@#f0?tlZiV zQBX*&%OlE0o;sllwfOC=g>A9bm|z5p2{>{ym_>6skB=b{#KfM+WF~n5RRC* zb8Wh15n;lngCPDx(`Sh6s?Ov^y4Of8E!RV`pj-4ta812N(y{GL{tAO+BkJ9s=OHB0 z?^l-G{#EDeVkn=&=k|2KWPoNx0g>e8P1g~ehxGZ{DMTpY=>YTySq(1^*j|Ea` zM}o+w(OD@K6_tPL|7$G16vMAI*Ord(;6Lo1X@eUDR=CocqVsGkXUXeFjTwL1FOP_= zRSWYCkFU3?een5-ddy8|29oJ}9nr}+4gxwWC9l3u2YZ`d+|suNE235v(^x(D2A zvD8`x1Na^gc!mQCVKNyMDR6MND7k*$LI;*AOwG8L_BPFa>fe)M#K7C1KV7OD@DvWv z5prujb;@1YI@Lw{>f5>0n+b%IX!3V|E+;|_A{mV7=F5D1?K-f7@UWwu&v&xDbnM8e zYvpFYJDiB|g07>{FJ2Jezqf*vFfMg(e5-{+{6M{An!S3G98pEE3iY37&HriLs#vn<>f+UI2H9(l@CjA4q zwy)b?H#$4?%%wXf8Et;tQysxCsL&dng4^sxhseuQkJ0x2Aqtr9NU)Zo4GoJy@>0y} z{&{x@Xcr{>oa{d-X&vy>UM2UYV|)mS=in(VRTjOY_-77`Jd2*$7(~wl>q@_f$TyeB z7Z}BZaZ+G=N=OwHy*REVU8-H+?%}TW-#42bRkr}#xB{GGP1X#kyq7$cH70*>?QZw~ zz#aJW|6s-VUvg6X{}-YEUmF4^;-TM0fNN`B^vCtGcY1lOO@Rvn<~da_08Us3G!JAY z+SvN~DksQeUY|ar^cY|@@7J5HKuB!Wb_qyx6U#jY|8%R9*Ir%_otB8bG6_Oybu*L+FVs~XFEvoRx5eRVcA^mA9q9#e$6cG$lE{e9<>&%X8RUrn9 zGJ2^%aU$X1413#(!I#{5E_UkIYTS6c{56f;kWd(FQu(!bgdb>z&=1hGE>_0m;*Ga( zG$LugO^^evagxK;6p0PGR$xrS{4~MXf};mAr~mfR@lImWO2?I2>o@DQt_)pUV6K;0 zcDnd@9Dt%e9e5D6>y5^+fd;}?1@&7D$nGz7t1WBh^yV2t+-eElFipL4us3LJ=pgkcM0{DUh`ecKdVx4F2o0Ha++j*hl_nr`h0kWj?Kt?erH#tEXxp7RuMoPc8;< zG}>%9DDP}7HN$u3jK>WMr?xk;ru9^Qv!8T|Bybs@O6EPJ$n1&U z<9#c2hsS2$aVFO>mW?AXk3T%z(K%hFVO8#gwg}ugr(Q`}$C$qx2|MXZ;%2QKZ#=+m z!H8i(er-y3TuzmvEhA;;lMz#p>+swutk;35zxsZ8GS##O=H5vQfi6L+ejG>Y8g{Ie z*Ee|eI|(>}Q@{UGK7G`*f>MsC_8Q9${umHAHO#Smf9ogBI|Ymjpt?P#e#cnSb=yQH24Ph~fl?LgM~)Nn4x5WK@7NBuq-`Wb5&iz&D@u z_QA%?FEV!@oH+u^>$PgXeQN?+!I3!Wd*Ue&B4$TbuK+dI)|PW^rnx$e>M)-5p$`se zW6mD9@`&ahin9YtfD_50VO z25Zkndo`Y(1NPX9w3WHKAgyQ6u#i0Or9=YJ5{PEdzB}58t^m9QSZ_Jf zm2WEve#DaP?+r$o6h^E}B8V?(;(GfxU^PH#K5Rxyg&L?C)=Y1(%_`N2+3b;K1Fu>$ zLA@8IxV%rL5sX%?#WTd=+!I3bfyrls4JkRjL93VR6REA2_uCLOR$fcY@4lyz6~P;u z1QeQ+lPIuzH|gcEWE7~35=QYSs^q<@W&?T3Hsr4B4<85iH&SgX`slGVFu*b0&(-p5 zOT1wOj4G8A2m%xXys*0Ct9Ik74WsJcol*cYHJ+a>h&^eMjiIf8j!~w<8rwEQZ`9Dm z8>%lV@K+A(Fo-{mfMD0mQ1hyI41Eohx_SRpyoTMoT0lxt$s zjG`J@bdX~;3vW-DyL*f6~pYq1cNZsIO<&ysFs8UnDb0?g9=Zpg6IT=~`eg3X|aeP7N zlTB&Nge|SZ*&BqQESv5n@HRkl{}%Y7bPOEM<~`Cnd>{R=eVa-8Yc~ryXRXUt4L-Yj|dYs{{lf5iA(UlfGw=D-TS&8IRNH zqa;k!U>UdJ8Wm%<=v3~~y-{WU^)|&Nu97r`>(sh(rFF%%VxsEZoT;zT1)$s`)ep+V z)scX@B+U14tziD~-tPV^|30l#_Bav3bWOEy`$Z@0X9kc8B?l)ck zo1uG2`!dX(PGDmXw#UY$NP~42a)p9d`;vNQ_3M+T<2p-J4SDWB2#|*lJbayOV-A(@ zxi@a!7?|cL^w0g(S$VS@g2X&{_>`>;7X#HnrHKK@NKlZ?539t&(+~v1xed=JZN2k4 z*<>g(8FBw5x#Y3riVt>@LjaZ4_VKdEIu6CM!fJ9dtN^HJQj}ClVkcWhM$H%8dZoTS?GBHU_pX$_DStx|q z1%exBb)Yc$uS5Xc$OS)iOi@Z_If?s{$nEv(4s{zBmlPK_08W3KphJnTM+k7s(go@3G?Gp=UZXF!Kb(7bYiDt)hL|hWvkNfvJCi;iS=0vkJjhd zRAjD5qfeSyVy!K50+iPCU3mA?c2&3i4*DQLl4T?E4WiX@5Dj9=t->y3p@R&lizK%` z33nCi5#IqXL2l3J(r{c&?br5n|A}&~%bzdf(?JY}>bIn(MPx=s#_$3}X{fP~E~(A| zrz(%lol9`~H0)mEP*{V_uO09}-R{M8{EMYg=w;q!+Jl4-?c53{*8?F(j~yF0(RF5D zuL)Wrv0lQ1c~c!LA|`vb@}qGY9)IJqwl=z5NkT%KIWv-GZsRX2e|`|s&!&a2zQ2`% zWlQuR0vJDBdhS_UV;z$im9w~QFgitAR@2@-<3S%h29Obu#0}oIwuynsK>?t3#<9MM z=l6;unY9+!I|0q;)LE1wwt?B>P*~SDxxS(pTs)A~8VAW@Mf$I_+5Gaty@OpIWojx^ zR6pz~C0e;Wn-G^mhQrw~RUuw@(5AM$k7Nz7hI=@n<6Bp@fBy2h7OFlX`X=x7&oBNX za8SHT-oCIic*M3Rsf|J;4qbQ$Am-ybyDJ&n$5p4pa7-!j5K)jMf$Rh$S#r-oZUUfp zUZmkK^L;{CSqgt|WAsC-%LV9BN6*n*AQoX8B)&P~s^Qp@yuoDU5S=1KSjSYqZB*5S z0oFEC=lNzqk>c_>ZSlI~g*qeAgRPaP2^dmdJsbRz$CK|C=+j%L1pL2@v-$^-?G(2( zhY@g$Q*Za^lF>4~)RD+r=sC_LN19vl_~AoFkNkG$?qU>J4<+adP>>NQbG7I~gtbzT zKeIA``YbRaLrpPr?%#-8<34>}xS=_p@$!<@8M@ePh1;9cr zBfqZ_J((+P%sDg8Oqm&a!cn+>-Bnydg2b_4jj6<$#cpL4)Q)eZ2`jh!>(rZHHOULV zo}({<%2<_M4EEqCO0FU~w=&D;@lD@An|Q)O&xelruiVeQE^oLzKJP8&L_0*`Xt5^m z{jK%F`;5VvrfVz~$xLB5)NKSuml3=-y^A}^qDaZeymU-Kw0O@K6pabEK$>~*OKxw5 zt$IqyMXmw;h4i!0ov@aVZYMpeGLeYu%J}ps{X;SE;ua^-1N-}7QzI4?T_TbEgCo0@ zggHG5+B(%l_yn~;a3F&UX<%`K&jf2!l6J&)9~sr$-28wgCMF?uTuF6T^rNVUjG&TN zLkDKxkEn3xIIUdaI#Arf2(S4UD#y|bX^EDiRGZoOhd=IQ+fT>VKMjArv&`a(5k&;) zkH8w!QsD?KqHA?T2RlDPneehKxl`u1hn-0(IC0+I>?(>9gMtO3_jvS&KA8EC*_msa z+{^9v$m=WtH)ltO>bkn;vC;qVT$$L80@Pz=7hOGF5U2!bPOlz_F>+&|h~glg$gPBl z=i4n(xf~2EAz{;FAIPF>ze(XLVq;^6sv>oG6%!*5LPLN;x?SCHSRmvq0<4OJJ$oLm za`P-tLd{1MvkzrqfidRp{k0dIiV`}nLp>kx-H2kMjObg#loW}jgz6uXHd z%rXs{!;;ERD2f||*zxQvG4$c~S*3r1yqT4(AS}(hTQGx=PV!Dt0dXro-woT69uJ2o zfRQ*DOLOjWh*7SEfSd_J3b<&-YzOfE+OiSY~k=&rfdfdM4@n#8i(En81vYNMEAba46er*)0jw z%)HU7<3JA-VP|y#8t~>5Oz*F-={OV)gU*8;%pMjkbpgH6Qd`$=9Sw68S=>04hYRf% z7Nui1ewjo&f-1bTN8=)fG|aOzB0Y<20x;3&v8B60P^O{EerE*u;iVIEQ-QfnpkbZ) zNz}uQ&u7Nya-$)YYA7%h>{UrgPjoqOBCe~{NvLe5O_hSFX_@Mn1X=CvO5i3EG4DcH z7{ao`FC*E{;S=s{1g`pDENpzjfeVs3S4;IFBQa_6n9m7JQjru?ANz~P%?{tKbtoB) zVJ7eV-tRf#iC>y8hb1-uDOxL7A;I{@n*~!-9Lsu&t)H5)^f~zffB?5YK8pks0Boi3~&2^z!aKKe< zLkFop{g%PfPyawpw|VNjF{z&ys7l+lly)A--fMKuLRq;3W(g9O4$tigegp`;yCCf` zDcLa$&W^MR$K*6=%6sj_MPHuhA&VhB3m%q!_fn}G?k7Z_g7p1`1FnZf+G!G>=?25eP+TXI*sS&9~I15 z`fdR}Np6j--H@a2bEWav_6`jY{J^LJK^1H>An(1Y&OwjnCW{hy<&`0RbOiyC{gmL3 z!5==|1GfZE9a#>7iQRv9EPfnz#LGH5br43^#I&&QE+roU-SCV@Z{L30T7{p>sXz)_ zfvpy1wlb7g?rS~VX{^7U+M8&1;AzCuShb@UgHnvdIjx@@3Z-Rb4Sjt}1y;G9d<(cwyt2h3wMI9eDDg@S@1MBA zlkj(4UL%p-J`sf8t| zBwRZOulP{rXFGD*NYTgxg$rCRJC&a0$5u?rOlY8YngQFzx13 Date: Mon, 3 Apr 2023 09:08:36 +0200 Subject: [PATCH 25/42] Update Helm README [skip ci] --- .github/workflows/helm.yml | 1 + helm/dendrite/README.md | 8 +++++++- helm/dendrite/values.yaml | 4 ++-- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/.github/workflows/helm.yml b/.github/workflows/helm.yml index 7cdc369ba2..a9c1718a04 100644 --- a/.github/workflows/helm.yml +++ b/.github/workflows/helm.yml @@ -6,6 +6,7 @@ on: - main paths: - 'helm/**' # only execute if we have helm chart changes + workflow_dispatch: jobs: release: diff --git a/helm/dendrite/README.md b/helm/dendrite/README.md index 90e31e040a..ca5705c036 100644 --- a/helm/dendrite/README.md +++ b/helm/dendrite/README.md @@ -1,7 +1,7 @@ # dendrite -![Version: 0.12.1](https://img.shields.io/badge/Version-0.12.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.12.0](https://img.shields.io/badge/AppVersion-0.12.0-informational?style=flat-square) +![Version: 0.12.2](https://img.shields.io/badge/Version-0.12.2-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.12.0](https://img.shields.io/badge/AppVersion-0.12.0-informational?style=flat-square) Dendrite Matrix Homeserver Status: **NOT PRODUCTION READY** @@ -55,6 +55,11 @@ Create a folder `appservices` and place your configurations in there. The confi | persistence.media.capacity | string | `"1Gi"` | PVC Storage Request for the media volume | | persistence.search.existingClaim | string | `""` | Use an existing volume claim for the fulltext search index | | persistence.search.capacity | string | `"1Gi"` | PVC Storage Request for the search volume | +| extraVolumes | list | `[]` | Add additional volumes to the Dendrite Pod | +| extraVolumeMounts | list | `[]` | Configure additional mount points volumes in the Dendrite Pod | +| strategy.type | string | `"RollingUpdate"` | Strategy to use for rolling updates (e.g. Recreate, RollingUpdate) If you are using ReadWriteOnce volumes, you should probably use Recreate | +| strategy.rollingUpdate.maxUnavailable | string | `"25%"` | Maximum number of pods that can be unavailable during the update process | +| strategy.rollingUpdate.maxSurge | string | `"25%"` | Maximum number of pods that can be scheduled above the desired number of pods | | dendrite_config.version | int | `2` | | | dendrite_config.global.server_name | string | `""` | **REQUIRED** Servername for this Dendrite deployment. | | dendrite_config.global.private_key | string | `"/etc/dendrite/secrets/signing.key"` | The private key to use. (**NOTE**: This is overriden in Helm) | @@ -181,3 +186,4 @@ grafana: enabled: true # will deploy default dashboards ``` PS: The label `release=kube-prometheus-stack` is setup with the helmchart of the Prometheus Operator. For Grafana Dashboards it may be necessary to enable scanning in the correct namespaces (or ALL), enabled by `sidecar.dashboards.searchNamespace` in [Helmchart of grafana](https://artifacthub.io/packages/helm/grafana/grafana) (which is part of PrometheusOperator, so `grafana.sidecar.dashboards.searchNamespace`) + diff --git a/helm/dendrite/values.yaml b/helm/dendrite/values.yaml index b0e8fc8a83..41ec1c3906 100644 --- a/helm/dendrite/values.yaml +++ b/helm/dendrite/values.yaml @@ -43,7 +43,7 @@ persistence: # -- PVC Storage Request for the search volume capacity: "1Gi" -# Add additional volumes to the Dendrite Pod +# -- Add additional volumes to the Dendrite Pod extraVolumes: [] # ex. # - name: extra-config @@ -51,7 +51,7 @@ extraVolumes: [] # secretName: extra-config -# Configure additional mount points volumes in the Dendrite Pod +# -- Configure additional mount points volumes in the Dendrite Pod extraVolumeMounts: [] # ex. # - mountPath: /etc/dendrite/extra-config From 10ef1fb11a547778b2ebec1b5cf9c4c743c76846 Mon Sep 17 00:00:00 2001 From: Till Faelligen <2353100+S7evinK@users.noreply.github.com> Date: Mon, 3 Apr 2023 15:08:33 +0200 Subject: [PATCH 26/42] Remove sync.Once when setting up server notice sender --- clientapi/routing/routing.go | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index 6c8035d400..58bea4ac70 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -18,7 +18,6 @@ import ( "context" "net/http" "strings" - "sync" "github.com/gorilla/mux" userapi "github.com/matrix-org/dendrite/userapi/api" @@ -198,18 +197,13 @@ func Setup( // server notifications if cfg.Matrix.ServerNotices.Enabled { logrus.Info("Enabling server notices at /_synapse/admin/v1/send_server_notice") - var serverNotificationSender *userapi.Device - var err error - notificationSenderOnce := &sync.Once{} + serverNotificationSender, err := getSenderDevice(context.Background(), rsAPI, userAPI, cfg) + if err != nil { + logrus.WithError(err).Fatal("unable to get account for sending sending server notices") + } synapseAdminRouter.Handle("/admin/v1/send_server_notice/{txnID}", httputil.MakeAuthAPI("send_server_notice", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - notificationSenderOnce.Do(func() { - serverNotificationSender, err = getSenderDevice(context.Background(), rsAPI, userAPI, cfg) - if err != nil { - logrus.WithError(err).Fatal("unable to get account for sending sending server notices") - } - }) // not specced, but ensure we're rate limiting requests to this endpoint if r := rateLimits.Limit(req, device); r != nil { return *r @@ -231,12 +225,6 @@ func Setup( synapseAdminRouter.Handle("/admin/v1/send_server_notice", httputil.MakeAuthAPI("send_server_notice", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - notificationSenderOnce.Do(func() { - serverNotificationSender, err = getSenderDevice(context.Background(), rsAPI, userAPI, cfg) - if err != nil { - logrus.WithError(err).Fatal("unable to get account for sending sending server notices") - } - }) // not specced, but ensure we're rate limiting requests to this endpoint if r := rateLimits.Limit(req, device); r != nil { return *r From 4cb9cd7842e58b542b25a2e9d7cdb7f61d147e96 Mon Sep 17 00:00:00 2001 From: Aiden Leong <1247008998@qq.com> Date: Tue, 4 Apr 2023 01:08:13 +0800 Subject: [PATCH 27/42] hard code path of README.md (#3035) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Pull Request Checklist * [x] I have added Go unit tests or [Complement integration tests](https://github.com/matrix-org/complement) for this PR _or_ I have justified why this PR doesn't need tests * [x] Pull request includes a [sign off below using a legally identifiable name](https://matrix-org.github.io/dendrite/development/contributing#sign-off) _or_ I have already signed off privately I was reading on https://matrix-org.github.io/dendrite/faq#is-dendrite-feature-complete. The link is dead due to relative path. 🍻 Signed-off-by: `Aiden Leong ` --- docs/FAQ.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/FAQ.md b/docs/FAQ.md index 2899aa9821..2000207265 100644 --- a/docs/FAQ.md +++ b/docs/FAQ.md @@ -18,7 +18,7 @@ Mostly, although there are still bugs and missing features. If you are a confide ## Is Dendrite feature-complete? -No, although a good portion of the Matrix specification has been implemented. Mostly missing are client features - see the [readme](../README.md) at the root of the repository for more information. +No, although a good portion of the Matrix specification has been implemented. Mostly missing are client features - see the [readme](https://github.com/matrix-org/dendrite/blob/main/README.md) at the root of the repository for more information. ## Why doesn't Dendrite have "x" yet? From c2db38d2954b8d5d1944b64671985d5ffd3b5c28 Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Mon, 3 Apr 2023 20:19:26 +0200 Subject: [PATCH 28/42] Add user profile tests, refactor user API methods (#3030) This adds tests for `/profile`. Also, as a first change in this regard, refactors the methods defined on the `UserInternalAPI` to not use structs as the request/response parameters. --- appservice/api/query.go | 36 ++-- clientapi/admin_test.go | 19 +++ clientapi/clientapi_test.go | 235 +++++++++++++++++++++++++- clientapi/routing/joinroom.go | 27 ++- clientapi/routing/profile.go | 87 ++++------ clientapi/routing/register.go | 8 +- clientapi/routing/register_test.go | 12 +- clientapi/routing/server_notices.go | 30 ++-- clientapi/threepid/invites.go | 11 +- federationapi/routing/profile.go | 19 +-- federationapi/routing/profile_test.go | 5 +- federationapi/routing/threepid.go | 9 +- internal/eventutil/types.go | 23 +-- test/user.go | 3 + userapi/api/api.go | 49 +----- userapi/internal/user_api.go | 37 ++-- userapi/userapi_test.go | 39 ++--- 17 files changed, 391 insertions(+), 258 deletions(-) diff --git a/appservice/api/query.go b/appservice/api/query.go index eb567b2ee2..472266d9ee 100644 --- a/appservice/api/query.go +++ b/appservice/api/query.go @@ -22,8 +22,6 @@ import ( "encoding/json" "errors" - "github.com/matrix-org/gomatrixserverlib" - "github.com/matrix-org/dendrite/clientapi/auth/authtypes" userapi "github.com/matrix-org/dendrite/userapi/api" ) @@ -150,6 +148,10 @@ type ASLocationResponse struct { Fields json.RawMessage `json:"fields"` } +// ErrProfileNotExists is returned when trying to lookup a user's profile that +// doesn't exist locally. +var ErrProfileNotExists = errors.New("no known profile for given user ID") + // RetrieveUserProfile is a wrapper that queries both the local database and // application services for a given user's profile // TODO: Remove this, it's called from federationapi and clientapi but is a pure function @@ -157,25 +159,11 @@ func RetrieveUserProfile( ctx context.Context, userID string, asAPI AppServiceInternalAPI, - profileAPI userapi.ClientUserAPI, + profileAPI userapi.ProfileAPI, ) (*authtypes.Profile, error) { - localpart, _, err := gomatrixserverlib.SplitID('@', userID) - if err != nil { - return nil, err - } - // Try to query the user from the local database - res := &userapi.QueryProfileResponse{} - err = profileAPI.QueryProfile(ctx, &userapi.QueryProfileRequest{UserID: userID}, res) - if err != nil { - return nil, err - } - profile := &authtypes.Profile{ - Localpart: localpart, - DisplayName: res.DisplayName, - AvatarURL: res.AvatarURL, - } - if res.UserExists { + profile, err := profileAPI.QueryProfile(ctx, userID) + if err == nil { return profile, nil } @@ -188,19 +176,15 @@ func RetrieveUserProfile( // If no user exists, return if !userResp.UserIDExists { - return nil, errors.New("no known profile for given user ID") + return nil, ErrProfileNotExists } // Try to query the user from the local database again - err = profileAPI.QueryProfile(ctx, &userapi.QueryProfileRequest{UserID: userID}, res) + profile, err = profileAPI.QueryProfile(ctx, userID) if err != nil { return nil, err } // profile should not be nil at this point - return &authtypes.Profile{ - Localpart: localpart, - DisplayName: res.DisplayName, - AvatarURL: res.AvatarURL, - }, nil + return profile, nil } diff --git a/clientapi/admin_test.go b/clientapi/admin_test.go index 79e88d137e..ca78f32e3d 100644 --- a/clientapi/admin_test.go +++ b/clientapi/admin_test.go @@ -261,6 +261,25 @@ func TestAdminEvacuateRoom(t *testing.T) { } }) } + + // Wait for the FS API to have consumed every message + js, _ := natsInstance.Prepare(processCtx, &cfg.Global.JetStream) + timeout := time.After(time.Second) + for { + select { + case <-timeout: + t.Fatalf("FS API didn't process all events in time") + default: + } + info, err := js.ConsumerInfo(cfg.Global.JetStream.Prefixed(jetstream.OutputRoomEvent), cfg.Global.JetStream.Durable("FederationAPIRoomServerConsumer")+"Pull") + if err != nil { + time.Sleep(time.Millisecond * 10) + continue + } + if info.NumPending == 0 && info.NumAckPending == 0 { + break + } + } }) } diff --git a/clientapi/clientapi_test.go b/clientapi/clientapi_test.go index d909155265..2168a2b3a1 100644 --- a/clientapi/clientapi_test.go +++ b/clientapi/clientapi_test.go @@ -4,25 +4,30 @@ import ( "bytes" "context" "encoding/json" + "fmt" + "io" "net/http" "net/http/httptest" "strings" "testing" "time" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/util" + "github.com/tidwall/gjson" + + "github.com/matrix-org/dendrite/appservice" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver" + "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/jetstream" "github.com/matrix-org/dendrite/test" "github.com/matrix-org/dendrite/test/testrig" "github.com/matrix-org/dendrite/userapi" uapi "github.com/matrix-org/dendrite/userapi/api" - "github.com/matrix-org/gomatrixserverlib" - "github.com/matrix-org/util" - "github.com/tidwall/gjson" ) type userDevice struct { @@ -371,3 +376,227 @@ func createAccessTokens(t *testing.T, accessTokens map[*test.User]userDevice, us } } } + +func TestSetDisplayname(t *testing.T) { + alice := test.NewUser(t) + bob := test.NewUser(t) + notLocalUser := &test.User{ID: "@charlie:localhost", Localpart: "charlie"} + changeDisplayName := "my new display name" + + testCases := []struct { + name string + user *test.User + wantOK bool + changeReq io.Reader + wantDisplayName string + }{ + { + name: "invalid user", + user: &test.User{ID: "!notauser"}, + }, + { + name: "non-existent user", + user: &test.User{ID: "@doesnotexist:test"}, + }, + { + name: "non-local user is not allowed", + user: notLocalUser, + }, + { + name: "existing user is allowed to change own name", + user: alice, + wantOK: true, + wantDisplayName: changeDisplayName, + }, + { + name: "existing user is not allowed to change own name if name is empty", + user: bob, + wantOK: false, + wantDisplayName: "", + }, + } + + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + cfg, processCtx, closeDB := testrig.CreateConfig(t, dbType) + defer closeDB() + caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) + routers := httputil.NewRouters() + cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) + natsInstance := &jetstream.NATSInstance{} + + rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, natsInstance, caches, caching.DisableMetrics) + rsAPI.SetFederationAPI(nil, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, natsInstance, rsAPI, nil) + asPI := appservice.NewInternalAPI(processCtx, cfg, natsInstance, userAPI, rsAPI) + + AddPublicRoutes(processCtx, routers, cfg, natsInstance, base.CreateFederationClient(cfg, nil), rsAPI, asPI, nil, nil, userAPI, nil, nil, caching.DisableMetrics) + + accessTokens := map[*test.User]userDevice{ + alice: {}, + bob: {}, + } + + createAccessTokens(t, accessTokens, userAPI, processCtx.Context(), routers) + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + wantDisplayName := tc.user.Localpart + if tc.changeReq == nil { + tc.changeReq = strings.NewReader("") + } + + // check profile after initial account creation + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/_matrix/client/v3/profile/"+tc.user.ID, strings.NewReader("")) + t.Logf("%s", req.URL.String()) + routers.Client.ServeHTTP(rec, req) + + if tc.wantOK && rec.Code != http.StatusOK { + t.Fatalf("expected HTTP 200, got %d", rec.Code) + } + + if gotDisplayName := gjson.GetBytes(rec.Body.Bytes(), "displayname").Str; tc.wantOK && gotDisplayName != wantDisplayName { + t.Fatalf("expected displayname to be '%s', but got '%s'", wantDisplayName, gotDisplayName) + } + + // now set the new display name + wantDisplayName = tc.wantDisplayName + tc.changeReq = strings.NewReader(fmt.Sprintf(`{"displayname":"%s"}`, tc.wantDisplayName)) + + rec = httptest.NewRecorder() + req = httptest.NewRequest(http.MethodPut, "/_matrix/client/v3/profile/"+tc.user.ID+"/displayname", tc.changeReq) + req.Header.Set("Authorization", "Bearer "+accessTokens[tc.user].accessToken) + + routers.Client.ServeHTTP(rec, req) + if tc.wantOK && rec.Code != http.StatusOK { + t.Fatalf("expected HTTP 200, got %d: %s", rec.Code, rec.Body.String()) + } + + // now only get the display name + rec = httptest.NewRecorder() + req = httptest.NewRequest(http.MethodGet, "/_matrix/client/v3/profile/"+tc.user.ID+"/displayname", strings.NewReader("")) + + routers.Client.ServeHTTP(rec, req) + if tc.wantOK && rec.Code != http.StatusOK { + t.Fatalf("expected HTTP 200, got %d: %s", rec.Code, rec.Body.String()) + } + + if gotDisplayName := gjson.GetBytes(rec.Body.Bytes(), "displayname").Str; tc.wantOK && gotDisplayName != wantDisplayName { + t.Fatalf("expected displayname to be '%s', but got '%s'", wantDisplayName, gotDisplayName) + } + }) + } + }) +} + +func TestSetAvatarURL(t *testing.T) { + alice := test.NewUser(t) + bob := test.NewUser(t) + notLocalUser := &test.User{ID: "@charlie:localhost", Localpart: "charlie"} + changeDisplayName := "mxc://newMXID" + + testCases := []struct { + name string + user *test.User + wantOK bool + changeReq io.Reader + avatar_url string + }{ + { + name: "invalid user", + user: &test.User{ID: "!notauser"}, + }, + { + name: "non-existent user", + user: &test.User{ID: "@doesnotexist:test"}, + }, + { + name: "non-local user is not allowed", + user: notLocalUser, + }, + { + name: "existing user is allowed to change own avatar", + user: alice, + wantOK: true, + avatar_url: changeDisplayName, + }, + { + name: "existing user is not allowed to change own avatar if avatar is empty", + user: bob, + wantOK: false, + avatar_url: "", + }, + } + + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + cfg, processCtx, closeDB := testrig.CreateConfig(t, dbType) + defer closeDB() + caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) + routers := httputil.NewRouters() + cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) + natsInstance := &jetstream.NATSInstance{} + + rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, natsInstance, caches, caching.DisableMetrics) + rsAPI.SetFederationAPI(nil, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, natsInstance, rsAPI, nil) + asPI := appservice.NewInternalAPI(processCtx, cfg, natsInstance, userAPI, rsAPI) + + AddPublicRoutes(processCtx, routers, cfg, natsInstance, base.CreateFederationClient(cfg, nil), rsAPI, asPI, nil, nil, userAPI, nil, nil, caching.DisableMetrics) + + accessTokens := map[*test.User]userDevice{ + alice: {}, + bob: {}, + } + + createAccessTokens(t, accessTokens, userAPI, processCtx.Context(), routers) + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + wantAvatarURL := "" + if tc.changeReq == nil { + tc.changeReq = strings.NewReader("") + } + + // check profile after initial account creation + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/_matrix/client/v3/profile/"+tc.user.ID, strings.NewReader("")) + t.Logf("%s", req.URL.String()) + routers.Client.ServeHTTP(rec, req) + + if tc.wantOK && rec.Code != http.StatusOK { + t.Fatalf("expected HTTP 200, got %d", rec.Code) + } + + if gotDisplayName := gjson.GetBytes(rec.Body.Bytes(), "avatar_url").Str; tc.wantOK && gotDisplayName != wantAvatarURL { + t.Fatalf("expected displayname to be '%s', but got '%s'", wantAvatarURL, gotDisplayName) + } + + // now set the new display name + wantAvatarURL = tc.avatar_url + tc.changeReq = strings.NewReader(fmt.Sprintf(`{"avatar_url":"%s"}`, tc.avatar_url)) + + rec = httptest.NewRecorder() + req = httptest.NewRequest(http.MethodPut, "/_matrix/client/v3/profile/"+tc.user.ID+"/avatar_url", tc.changeReq) + req.Header.Set("Authorization", "Bearer "+accessTokens[tc.user].accessToken) + + routers.Client.ServeHTTP(rec, req) + if tc.wantOK && rec.Code != http.StatusOK { + t.Fatalf("expected HTTP 200, got %d: %s", rec.Code, rec.Body.String()) + } + + // now only get the display name + rec = httptest.NewRecorder() + req = httptest.NewRequest(http.MethodGet, "/_matrix/client/v3/profile/"+tc.user.ID+"/avatar_url", strings.NewReader("")) + + routers.Client.ServeHTTP(rec, req) + if tc.wantOK && rec.Code != http.StatusOK { + t.Fatalf("expected HTTP 200, got %d: %s", rec.Code, rec.Body.String()) + } + + if gotDisplayName := gjson.GetBytes(rec.Body.Bytes(), "avatar_url").Str; tc.wantOK && gotDisplayName != wantAvatarURL { + t.Fatalf("expected displayname to be '%s', but got '%s'", wantAvatarURL, gotDisplayName) + } + }) + } + }) +} diff --git a/clientapi/routing/joinroom.go b/clientapi/routing/joinroom.go index e371d92148..3493dd6d88 100644 --- a/clientapi/routing/joinroom.go +++ b/clientapi/routing/joinroom.go @@ -18,6 +18,7 @@ import ( "net/http" "time" + appserviceAPI "github.com/matrix-org/dendrite/appservice/api" "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" @@ -61,21 +62,19 @@ func JoinRoomByIDOrAlias( // Work out our localpart for the client profile request. // Request our profile content to populate the request content with. - res := &api.QueryProfileResponse{} - err := profileAPI.QueryProfile(req.Context(), &api.QueryProfileRequest{UserID: device.UserID}, res) - if err != nil || !res.UserExists { - if !res.UserExists { - util.GetLogger(req.Context()).Error("Unable to query user profile, no profile found.") - return util.JSONResponse{ - Code: http.StatusInternalServerError, - JSON: jsonerror.Unknown("Unable to query user profile, no profile found."), - } - } + profile, err := profileAPI.QueryProfile(req.Context(), device.UserID) - util.GetLogger(req.Context()).WithError(err).Error("UserProfileAPI.QueryProfile failed") - } else { - joinReq.Content["displayname"] = res.DisplayName - joinReq.Content["avatar_url"] = res.AvatarURL + switch err { + case nil: + joinReq.Content["displayname"] = profile.DisplayName + joinReq.Content["avatar_url"] = profile.AvatarURL + case appserviceAPI.ErrProfileNotExists: + util.GetLogger(req.Context()).Error("Unable to query user profile, no profile found.") + return util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: jsonerror.Unknown("Unable to query user profile, no profile found."), + } + default: } // Ask the roomserver to perform the join. diff --git a/clientapi/routing/profile.go b/clientapi/routing/profile.go index 92a75fc781..ab0fd990ef 100644 --- a/clientapi/routing/profile.go +++ b/clientapi/routing/profile.go @@ -36,14 +36,14 @@ import ( // GetProfile implements GET /profile/{userID} func GetProfile( - req *http.Request, profileAPI userapi.ClientUserAPI, cfg *config.ClientAPI, + req *http.Request, profileAPI userapi.ProfileAPI, cfg *config.ClientAPI, userID string, asAPI appserviceAPI.AppServiceInternalAPI, federation *gomatrixserverlib.FederationClient, ) util.JSONResponse { profile, err := getProfile(req.Context(), profileAPI, cfg, userID, asAPI, federation) if err != nil { - if err == eventutil.ErrProfileNoExists { + if err == appserviceAPI.ErrProfileNotExists { return util.JSONResponse{ Code: http.StatusNotFound, JSON: jsonerror.NotFound("The user does not exist or does not have a profile"), @@ -56,7 +56,7 @@ func GetProfile( return util.JSONResponse{ Code: http.StatusOK, - JSON: eventutil.ProfileResponse{ + JSON: eventutil.UserProfile{ AvatarURL: profile.AvatarURL, DisplayName: profile.DisplayName, }, @@ -65,34 +65,28 @@ func GetProfile( // GetAvatarURL implements GET /profile/{userID}/avatar_url func GetAvatarURL( - req *http.Request, profileAPI userapi.ClientUserAPI, cfg *config.ClientAPI, + req *http.Request, profileAPI userapi.ProfileAPI, cfg *config.ClientAPI, userID string, asAPI appserviceAPI.AppServiceInternalAPI, federation *gomatrixserverlib.FederationClient, ) util.JSONResponse { - profile, err := getProfile(req.Context(), profileAPI, cfg, userID, asAPI, federation) - if err != nil { - if err == eventutil.ErrProfileNoExists { - return util.JSONResponse{ - Code: http.StatusNotFound, - JSON: jsonerror.NotFound("The user does not exist or does not have a profile"), - } - } - - util.GetLogger(req.Context()).WithError(err).Error("getProfile failed") - return jsonerror.InternalServerError() + profile := GetProfile(req, profileAPI, cfg, userID, asAPI, federation) + p, ok := profile.JSON.(eventutil.UserProfile) + // not a profile response, so most likely an error, return that + if !ok { + return profile } return util.JSONResponse{ Code: http.StatusOK, - JSON: eventutil.AvatarURL{ - AvatarURL: profile.AvatarURL, + JSON: eventutil.UserProfile{ + AvatarURL: p.AvatarURL, }, } } // SetAvatarURL implements PUT /profile/{userID}/avatar_url func SetAvatarURL( - req *http.Request, profileAPI userapi.ClientUserAPI, + req *http.Request, profileAPI userapi.ProfileAPI, device *userapi.Device, userID string, cfg *config.ClientAPI, rsAPI api.ClientRoomserverAPI, ) util.JSONResponse { if userID != device.UserID { @@ -102,7 +96,7 @@ func SetAvatarURL( } } - var r eventutil.AvatarURL + var r eventutil.UserProfile if resErr := httputil.UnmarshalJSONRequest(req, &r); resErr != nil { return *resErr } @@ -134,24 +128,20 @@ func SetAvatarURL( } } - setRes := &userapi.PerformSetAvatarURLResponse{} - if err = profileAPI.SetAvatarURL(req.Context(), &userapi.PerformSetAvatarURLRequest{ - Localpart: localpart, - ServerName: domain, - AvatarURL: r.AvatarURL, - }, setRes); err != nil { + profile, changed, err := profileAPI.SetAvatarURL(req.Context(), localpart, domain, r.AvatarURL) + if err != nil { util.GetLogger(req.Context()).WithError(err).Error("profileAPI.SetAvatarURL failed") return jsonerror.InternalServerError() } // No need to build new membership events, since nothing changed - if !setRes.Changed { + if !changed { return util.JSONResponse{ Code: http.StatusOK, JSON: struct{}{}, } } - response, err := updateProfile(req.Context(), rsAPI, device, setRes.Profile, userID, cfg, evTime) + response, err := updateProfile(req.Context(), rsAPI, device, profile, userID, cfg, evTime) if err != nil { return response } @@ -164,34 +154,28 @@ func SetAvatarURL( // GetDisplayName implements GET /profile/{userID}/displayname func GetDisplayName( - req *http.Request, profileAPI userapi.ClientUserAPI, cfg *config.ClientAPI, + req *http.Request, profileAPI userapi.ProfileAPI, cfg *config.ClientAPI, userID string, asAPI appserviceAPI.AppServiceInternalAPI, federation *gomatrixserverlib.FederationClient, ) util.JSONResponse { - profile, err := getProfile(req.Context(), profileAPI, cfg, userID, asAPI, federation) - if err != nil { - if err == eventutil.ErrProfileNoExists { - return util.JSONResponse{ - Code: http.StatusNotFound, - JSON: jsonerror.NotFound("The user does not exist or does not have a profile"), - } - } - - util.GetLogger(req.Context()).WithError(err).Error("getProfile failed") - return jsonerror.InternalServerError() + profile := GetProfile(req, profileAPI, cfg, userID, asAPI, federation) + p, ok := profile.JSON.(eventutil.UserProfile) + // not a profile response, so most likely an error, return that + if !ok { + return profile } return util.JSONResponse{ Code: http.StatusOK, - JSON: eventutil.DisplayName{ - DisplayName: profile.DisplayName, + JSON: eventutil.UserProfile{ + DisplayName: p.DisplayName, }, } } // SetDisplayName implements PUT /profile/{userID}/displayname func SetDisplayName( - req *http.Request, profileAPI userapi.ClientUserAPI, + req *http.Request, profileAPI userapi.ProfileAPI, device *userapi.Device, userID string, cfg *config.ClientAPI, rsAPI api.ClientRoomserverAPI, ) util.JSONResponse { if userID != device.UserID { @@ -201,7 +185,7 @@ func SetDisplayName( } } - var r eventutil.DisplayName + var r eventutil.UserProfile if resErr := httputil.UnmarshalJSONRequest(req, &r); resErr != nil { return *resErr } @@ -233,25 +217,20 @@ func SetDisplayName( } } - profileRes := &userapi.PerformUpdateDisplayNameResponse{} - err = profileAPI.SetDisplayName(req.Context(), &userapi.PerformUpdateDisplayNameRequest{ - Localpart: localpart, - ServerName: domain, - DisplayName: r.DisplayName, - }, profileRes) + profile, changed, err := profileAPI.SetDisplayName(req.Context(), localpart, domain, r.DisplayName) if err != nil { util.GetLogger(req.Context()).WithError(err).Error("profileAPI.SetDisplayName failed") return jsonerror.InternalServerError() } // No need to build new membership events, since nothing changed - if !profileRes.Changed { + if !changed { return util.JSONResponse{ Code: http.StatusOK, JSON: struct{}{}, } } - response, err := updateProfile(req.Context(), rsAPI, device, profileRes.Profile, userID, cfg, evTime) + response, err := updateProfile(req.Context(), rsAPI, device, profile, userID, cfg, evTime) if err != nil { return response } @@ -308,9 +287,9 @@ func updateProfile( // getProfile gets the full profile of a user by querying the database or a // remote homeserver. // Returns an error when something goes wrong or specifically -// eventutil.ErrProfileNoExists when the profile doesn't exist. +// eventutil.ErrProfileNotExists when the profile doesn't exist. func getProfile( - ctx context.Context, profileAPI userapi.ClientUserAPI, cfg *config.ClientAPI, + ctx context.Context, profileAPI userapi.ProfileAPI, cfg *config.ClientAPI, userID string, asAPI appserviceAPI.AppServiceInternalAPI, federation *gomatrixserverlib.FederationClient, @@ -325,7 +304,7 @@ func getProfile( if fedErr != nil { if x, ok := fedErr.(gomatrix.HTTPError); ok { if x.Code == http.StatusNotFound { - return nil, eventutil.ErrProfileNoExists + return nil, appserviceAPI.ErrProfileNotExists } } diff --git a/clientapi/routing/register.go b/clientapi/routing/register.go index ff6a0900e2..d880961f98 100644 --- a/clientapi/routing/register.go +++ b/clientapi/routing/register.go @@ -888,13 +888,7 @@ func completeRegistration( } if displayName != "" { - nameReq := userapi.PerformUpdateDisplayNameRequest{ - Localpart: username, - ServerName: serverName, - DisplayName: displayName, - } - var nameRes userapi.PerformUpdateDisplayNameResponse - err = userAPI.SetDisplayName(ctx, &nameReq, &nameRes) + _, _, err = userAPI.SetDisplayName(ctx, username, serverName, displayName) if err != nil { return util.JSONResponse{ Code: http.StatusInternalServerError, diff --git a/clientapi/routing/register_test.go b/clientapi/routing/register_test.go index 46cd8b2b90..b07f636dd9 100644 --- a/clientapi/routing/register_test.go +++ b/clientapi/routing/register_test.go @@ -611,11 +611,9 @@ func TestRegisterUserWithDisplayName(t *testing.T) { assert.Equal(t, http.StatusOK, response.Code) - req := api.QueryProfileRequest{UserID: "@user:server"} - var res api.QueryProfileResponse - err := userAPI.QueryProfile(processCtx.Context(), &req, &res) + profile, err := userAPI.QueryProfile(processCtx.Context(), "@user:server") assert.NoError(t, err) - assert.Equal(t, expectedDisplayName, res.DisplayName) + assert.Equal(t, expectedDisplayName, profile.DisplayName) }) } @@ -662,10 +660,8 @@ func TestRegisterAdminUsingSharedSecret(t *testing.T) { ) assert.Equal(t, http.StatusOK, response.Code) - profilReq := api.QueryProfileRequest{UserID: "@alice:server"} - var profileRes api.QueryProfileResponse - err = userAPI.QueryProfile(processCtx.Context(), &profilReq, &profileRes) + profile, err := userAPI.QueryProfile(processCtx.Context(), "@alice:server") assert.NoError(t, err) - assert.Equal(t, expectedDisplayName, profileRes.DisplayName) + assert.Equal(t, expectedDisplayName, profile.DisplayName) }) } diff --git a/clientapi/routing/server_notices.go b/clientapi/routing/server_notices.go index fb93d87834..d6191f3b42 100644 --- a/clientapi/routing/server_notices.go +++ b/clientapi/routing/server_notices.go @@ -295,30 +295,28 @@ func getSenderDevice( } // Set the avatarurl for the user - avatarRes := &userapi.PerformSetAvatarURLResponse{} - if err = userAPI.SetAvatarURL(ctx, &userapi.PerformSetAvatarURLRequest{ - Localpart: cfg.Matrix.ServerNotices.LocalPart, - ServerName: cfg.Matrix.ServerName, - AvatarURL: cfg.Matrix.ServerNotices.AvatarURL, - }, avatarRes); err != nil { + profile, avatarChanged, err := userAPI.SetAvatarURL(ctx, + cfg.Matrix.ServerNotices.LocalPart, + cfg.Matrix.ServerName, + cfg.Matrix.ServerNotices.AvatarURL, + ) + if err != nil { util.GetLogger(ctx).WithError(err).Error("userAPI.SetAvatarURL failed") return nil, err } - profile := avatarRes.Profile - // Set the displayname for the user - displayNameRes := &userapi.PerformUpdateDisplayNameResponse{} - if err = userAPI.SetDisplayName(ctx, &userapi.PerformUpdateDisplayNameRequest{ - Localpart: cfg.Matrix.ServerNotices.LocalPart, - ServerName: cfg.Matrix.ServerName, - DisplayName: cfg.Matrix.ServerNotices.DisplayName, - }, displayNameRes); err != nil { + _, displayNameChanged, err := userAPI.SetDisplayName(ctx, + cfg.Matrix.ServerNotices.LocalPart, + cfg.Matrix.ServerName, + cfg.Matrix.ServerNotices.DisplayName, + ) + if err != nil { util.GetLogger(ctx).WithError(err).Error("userAPI.SetDisplayName failed") return nil, err } - if displayNameRes.Changed { + if displayNameChanged { profile.DisplayName = cfg.Matrix.ServerNotices.DisplayName } @@ -334,7 +332,7 @@ func getSenderDevice( // We've got an existing account, return the first device of it if len(deviceRes.Devices) > 0 { // If there were changes to the profile, create a new membership event - if displayNameRes.Changed || avatarRes.Changed { + if displayNameChanged || avatarChanged { _, err = updateProfile(ctx, rsAPI, &deviceRes.Devices[0], profile, accRes.Account.UserID, cfg, time.Now()) if err != nil { return nil, err diff --git a/clientapi/threepid/invites.go b/clientapi/threepid/invites.go index 1f294a032b..a9910b782a 100644 --- a/clientapi/threepid/invites.go +++ b/clientapi/threepid/invites.go @@ -209,24 +209,17 @@ func queryIDServerStoreInvite( body *MembershipRequest, roomID string, ) (*idServerStoreInviteResponse, error) { // Retrieve the sender's profile to get their display name - localpart, serverName, err := gomatrixserverlib.SplitID('@', device.UserID) + _, serverName, err := gomatrixserverlib.SplitID('@', device.UserID) if err != nil { return nil, err } var profile *authtypes.Profile if cfg.Matrix.IsLocalServerName(serverName) { - res := &userapi.QueryProfileResponse{} - err = userAPI.QueryProfile(ctx, &userapi.QueryProfileRequest{UserID: device.UserID}, res) + profile, err = userAPI.QueryProfile(ctx, device.UserID) if err != nil { return nil, err } - profile = &authtypes.Profile{ - Localpart: localpart, - DisplayName: res.DisplayName, - AvatarURL: res.AvatarURL, - } - } else { profile = &authtypes.Profile{} } diff --git a/federationapi/routing/profile.go b/federationapi/routing/profile.go index e4d2230ad2..55641b216e 100644 --- a/federationapi/routing/profile.go +++ b/federationapi/routing/profile.go @@ -50,10 +50,7 @@ func GetProfile( } } - var profileRes userapi.QueryProfileResponse - err = userAPI.QueryProfile(httpReq.Context(), &userapi.QueryProfileRequest{ - UserID: userID, - }, &profileRes) + profile, err := userAPI.QueryProfile(httpReq.Context(), userID) if err != nil { util.GetLogger(httpReq.Context()).WithError(err).Error("userAPI.QueryProfile failed") return jsonerror.InternalServerError() @@ -65,21 +62,21 @@ func GetProfile( if field != "" { switch field { case "displayname": - res = eventutil.DisplayName{ - DisplayName: profileRes.DisplayName, + res = eventutil.UserProfile{ + DisplayName: profile.DisplayName, } case "avatar_url": - res = eventutil.AvatarURL{ - AvatarURL: profileRes.AvatarURL, + res = eventutil.UserProfile{ + AvatarURL: profile.AvatarURL, } default: code = http.StatusBadRequest res = jsonerror.InvalidArgumentValue("The request body did not contain an allowed value of argument 'field'. Allowed values are either: 'avatar_url', 'displayname'.") } } else { - res = eventutil.ProfileResponse{ - AvatarURL: profileRes.AvatarURL, - DisplayName: profileRes.DisplayName, + res = eventutil.UserProfile{ + AvatarURL: profile.AvatarURL, + DisplayName: profile.DisplayName, } } diff --git a/federationapi/routing/profile_test.go b/federationapi/routing/profile_test.go index d5e9997faf..d249fce14f 100644 --- a/federationapi/routing/profile_test.go +++ b/federationapi/routing/profile_test.go @@ -23,6 +23,7 @@ import ( "testing" "github.com/gorilla/mux" + "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing" fedAPI "github.com/matrix-org/dendrite/federationapi" fedInternal "github.com/matrix-org/dendrite/federationapi/internal" @@ -43,8 +44,8 @@ type fakeUserAPI struct { userAPI.FederationUserAPI } -func (u *fakeUserAPI) QueryProfile(ctx context.Context, req *userAPI.QueryProfileRequest, res *userAPI.QueryProfileResponse) error { - return nil +func (u *fakeUserAPI) QueryProfile(ctx context.Context, userID string) (*authtypes.Profile, error) { + return &authtypes.Profile{}, nil } func TestHandleQueryProfile(t *testing.T) { diff --git a/federationapi/routing/threepid.go b/federationapi/routing/threepid.go index d07faef39f..048183ad14 100644 --- a/federationapi/routing/threepid.go +++ b/federationapi/routing/threepid.go @@ -256,17 +256,14 @@ func createInviteFrom3PIDInvite( StateKey: &inv.MXID, } - var res userapi.QueryProfileResponse - err = userAPI.QueryProfile(ctx, &userapi.QueryProfileRequest{ - UserID: inv.MXID, - }, &res) + profile, err := userAPI.QueryProfile(ctx, inv.MXID) if err != nil { return nil, err } content := gomatrixserverlib.MemberContent{ - AvatarURL: res.AvatarURL, - DisplayName: res.DisplayName, + AvatarURL: profile.AvatarURL, + DisplayName: profile.DisplayName, Membership: gomatrixserverlib.Invite, ThirdPartyInvite: &gomatrixserverlib.MemberThirdPartyInvite{ Signed: inv.Signed, diff --git a/internal/eventutil/types.go b/internal/eventutil/types.go index 18175d6a0c..e43c0412e4 100644 --- a/internal/eventutil/types.go +++ b/internal/eventutil/types.go @@ -15,16 +15,11 @@ package eventutil import ( - "errors" "strconv" "github.com/matrix-org/dendrite/syncapi/types" ) -// ErrProfileNoExists is returned when trying to lookup a user's profile that -// doesn't exist locally. -var ErrProfileNoExists = errors.New("no known profile for given user ID") - // AccountData represents account data sent from the client API server to the // sync API server type AccountData struct { @@ -56,20 +51,10 @@ type NotificationData struct { UnreadNotificationCount int `json:"unread_notification_count"` } -// ProfileResponse is a struct containing all known user profile data -type ProfileResponse struct { - AvatarURL string `json:"avatar_url"` - DisplayName string `json:"displayname"` -} - -// AvatarURL is a struct containing only the URL to a user's avatar -type AvatarURL struct { - AvatarURL string `json:"avatar_url"` -} - -// DisplayName is a struct containing only a user's display name -type DisplayName struct { - DisplayName string `json:"displayname"` +// UserProfile is a struct containing all known user profile data +type UserProfile struct { + AvatarURL string `json:"avatar_url,omitempty"` + DisplayName string `json:"displayname,omitempty"` } // WeakBoolean is a type that will Unmarshal to true or false even if the encoded diff --git a/test/user.go b/test/user.go index 95a8f83e6f..63206fa162 100644 --- a/test/user.go +++ b/test/user.go @@ -17,6 +17,7 @@ package test import ( "crypto/ed25519" "fmt" + "strconv" "sync/atomic" "testing" @@ -47,6 +48,7 @@ var ( type User struct { ID string + Localpart string AccountType api.AccountType // key ID and private key of the server who has this user, if known. keyID gomatrixserverlib.KeyID @@ -81,6 +83,7 @@ func NewUser(t *testing.T, opts ...UserOpt) *User { WithSigningServer(serverName, keyID, privateKey)(&u) } u.ID = fmt.Sprintf("@%d:%s", counter, u.srvName) + u.Localpart = strconv.Itoa(int(counter)) t.Logf("NewUser: created user %s", u.ID) return &u } diff --git a/userapi/api/api.go b/userapi/api/api.go index 19d4868483..5fd7992c8a 100644 --- a/userapi/api/api.go +++ b/userapi/api/api.go @@ -58,7 +58,7 @@ type MediaUserAPI interface { type FederationUserAPI interface { UploadDeviceKeysAPI QueryOpenIDToken(ctx context.Context, req *QueryOpenIDTokenRequest, res *QueryOpenIDTokenResponse) error - QueryProfile(ctx context.Context, req *QueryProfileRequest, res *QueryProfileResponse) error + QueryProfile(ctx context.Context, userID string) (*authtypes.Profile, error) QueryDevices(ctx context.Context, req *QueryDevicesRequest, res *QueryDevicesResponse) error QueryKeys(ctx context.Context, req *QueryKeysRequest, res *QueryKeysResponse) error QuerySignatures(ctx context.Context, req *QuerySignaturesRequest, res *QuerySignaturesResponse) error @@ -83,9 +83,9 @@ type ClientUserAPI interface { LoginTokenInternalAPI UserLoginAPI ClientKeyAPI + ProfileAPI QueryNumericLocalpart(ctx context.Context, req *QueryNumericLocalpartRequest, res *QueryNumericLocalpartResponse) error QueryDevices(ctx context.Context, req *QueryDevicesRequest, res *QueryDevicesResponse) error - QueryProfile(ctx context.Context, req *QueryProfileRequest, res *QueryProfileResponse) error QueryAccountData(ctx context.Context, req *QueryAccountDataRequest, res *QueryAccountDataResponse) error QueryPushers(ctx context.Context, req *QueryPushersRequest, res *QueryPushersResponse) error QueryPushRules(ctx context.Context, req *QueryPushRulesRequest, res *QueryPushRulesResponse) error @@ -100,8 +100,6 @@ type ClientUserAPI interface { PerformPushRulesPut(ctx context.Context, req *PerformPushRulesPutRequest, res *struct{}) error PerformAccountDeactivation(ctx context.Context, req *PerformAccountDeactivationRequest, res *PerformAccountDeactivationResponse) error PerformOpenIDTokenCreation(ctx context.Context, req *PerformOpenIDTokenCreationRequest, res *PerformOpenIDTokenCreationResponse) error - SetAvatarURL(ctx context.Context, req *PerformSetAvatarURLRequest, res *PerformSetAvatarURLResponse) error - SetDisplayName(ctx context.Context, req *PerformUpdateDisplayNameRequest, res *PerformUpdateDisplayNameResponse) error QueryNotifications(ctx context.Context, req *QueryNotificationsRequest, res *QueryNotificationsResponse) error InputAccountData(ctx context.Context, req *InputAccountDataRequest, res *InputAccountDataResponse) error PerformKeyBackup(ctx context.Context, req *PerformKeyBackupRequest, res *PerformKeyBackupResponse) error @@ -113,6 +111,12 @@ type ClientUserAPI interface { PerformSaveThreePIDAssociation(ctx context.Context, req *PerformSaveThreePIDAssociationRequest, res *struct{}) error } +type ProfileAPI interface { + QueryProfile(ctx context.Context, userID string) (*authtypes.Profile, error) + SetAvatarURL(ctx context.Context, localpart string, serverName gomatrixserverlib.ServerName, avatarURL string) (*authtypes.Profile, bool, error) + SetDisplayName(ctx context.Context, localpart string, serverName gomatrixserverlib.ServerName, displayName string) (*authtypes.Profile, bool, error) +} + // custom api functions required by pinecone / p2p demos type QuerySearchProfilesAPI interface { QuerySearchProfiles(ctx context.Context, req *QuerySearchProfilesRequest, res *QuerySearchProfilesResponse) error @@ -290,22 +294,6 @@ type QueryDevicesResponse struct { Devices []Device } -// QueryProfileRequest is the request for QueryProfile -type QueryProfileRequest struct { - // The user ID to query - UserID string -} - -// QueryProfileResponse is the response for QueryProfile -type QueryProfileResponse struct { - // True if the user exists. Querying for a profile does not create them. - UserExists bool - // The current display name if set. - DisplayName string - // The current avatar URL if set. - AvatarURL string -} - // QuerySearchProfilesRequest is the request for QueryProfile type QuerySearchProfilesRequest struct { // The search string to match @@ -600,16 +588,6 @@ type Notification struct { TS gomatrixserverlib.Timestamp `json:"ts"` // Required. } -type PerformSetAvatarURLRequest struct { - Localpart string - ServerName gomatrixserverlib.ServerName - AvatarURL string -} -type PerformSetAvatarURLResponse struct { - Profile *authtypes.Profile `json:"profile"` - Changed bool `json:"changed"` -} - type QueryNumericLocalpartRequest struct { ServerName gomatrixserverlib.ServerName } @@ -638,17 +616,6 @@ type QueryAccountByPasswordResponse struct { Exists bool } -type PerformUpdateDisplayNameRequest struct { - Localpart string - ServerName gomatrixserverlib.ServerName - DisplayName string -} - -type PerformUpdateDisplayNameResponse struct { - Profile *authtypes.Profile `json:"profile"` - Changed bool `json:"changed"` -} - type QueryLocalpartForThreePIDRequest struct { ThreePID, Medium string } diff --git a/userapi/internal/user_api.go b/userapi/internal/user_api.go index 4049d13b61..6dad91dd9c 100644 --- a/userapi/internal/user_api.go +++ b/userapi/internal/user_api.go @@ -23,6 +23,8 @@ import ( "strconv" "time" + appserviceAPI "github.com/matrix-org/dendrite/appservice/api" + "github.com/matrix-org/dendrite/clientapi/auth/authtypes" fedsenderapi "github.com/matrix-org/dendrite/federationapi/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" @@ -418,25 +420,26 @@ func (a *UserInternalAPI) PerformDeviceUpdate(ctx context.Context, req *api.Perf return nil } -func (a *UserInternalAPI) QueryProfile(ctx context.Context, req *api.QueryProfileRequest, res *api.QueryProfileResponse) error { - local, domain, err := gomatrixserverlib.SplitID('@', req.UserID) +var ( + ErrIsRemoteServer = errors.New("cannot query profile of remote users") +) + +func (a *UserInternalAPI) QueryProfile(ctx context.Context, userID string) (*authtypes.Profile, error) { + local, domain, err := gomatrixserverlib.SplitID('@', userID) if err != nil { - return err + return nil, err } if !a.Config.Matrix.IsLocalServerName(domain) { - return fmt.Errorf("cannot query profile of remote users (server name %s)", domain) + return nil, ErrIsRemoteServer } prof, err := a.DB.GetProfileByLocalpart(ctx, local, domain) if err != nil { if err == sql.ErrNoRows { - return nil + return nil, appserviceAPI.ErrProfileNotExists } - return err + return nil, err } - res.UserExists = true - res.AvatarURL = prof.AvatarURL - res.DisplayName = prof.DisplayName - return nil + return prof, nil } func (a *UserInternalAPI) QuerySearchProfiles(ctx context.Context, req *api.QuerySearchProfilesRequest, res *api.QuerySearchProfilesResponse) error { @@ -901,11 +904,8 @@ func (a *UserInternalAPI) QueryPushRules(ctx context.Context, req *api.QueryPush return nil } -func (a *UserInternalAPI) SetAvatarURL(ctx context.Context, req *api.PerformSetAvatarURLRequest, res *api.PerformSetAvatarURLResponse) error { - profile, changed, err := a.DB.SetAvatarURL(ctx, req.Localpart, req.ServerName, req.AvatarURL) - res.Profile = profile - res.Changed = changed - return err +func (a *UserInternalAPI) SetAvatarURL(ctx context.Context, localpart string, serverName gomatrixserverlib.ServerName, avatarURL string) (*authtypes.Profile, bool, error) { + return a.DB.SetAvatarURL(ctx, localpart, serverName, avatarURL) } func (a *UserInternalAPI) QueryNumericLocalpart(ctx context.Context, req *api.QueryNumericLocalpartRequest, res *api.QueryNumericLocalpartResponse) error { @@ -939,11 +939,8 @@ func (a *UserInternalAPI) QueryAccountByPassword(ctx context.Context, req *api.Q } } -func (a *UserInternalAPI) SetDisplayName(ctx context.Context, req *api.PerformUpdateDisplayNameRequest, res *api.PerformUpdateDisplayNameResponse) error { - profile, changed, err := a.DB.SetDisplayName(ctx, req.Localpart, req.ServerName, req.DisplayName) - res.Profile = profile - res.Changed = changed - return err +func (a *UserInternalAPI) SetDisplayName(ctx context.Context, localpart string, serverName gomatrixserverlib.ServerName, displayName string) (*authtypes.Profile, bool, error) { + return a.DB.SetDisplayName(ctx, localpart, serverName, displayName) } func (a *UserInternalAPI) QueryLocalpartForThreePID(ctx context.Context, req *api.QueryLocalpartForThreePIDRequest, res *api.QueryLocalpartForThreePIDResponse) error { diff --git a/userapi/userapi_test.go b/userapi/userapi_test.go index 03e6563545..e29246ec14 100644 --- a/userapi/userapi_test.go +++ b/userapi/userapi_test.go @@ -22,6 +22,8 @@ import ( "testing" "time" + api2 "github.com/matrix-org/dendrite/appservice/api" + "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/userapi/producers" "github.com/matrix-org/gomatrixserverlib" @@ -111,33 +113,26 @@ func TestQueryProfile(t *testing.T) { aliceDisplayName := "Alice" testCases := []struct { - req api.QueryProfileRequest - wantRes api.QueryProfileResponse + userID string + wantRes *authtypes.Profile wantErr error }{ { - req: api.QueryProfileRequest{ - UserID: fmt.Sprintf("@alice:%s", serverName), - }, - wantRes: api.QueryProfileResponse{ - UserExists: true, - AvatarURL: aliceAvatarURL, + userID: fmt.Sprintf("@alice:%s", serverName), + wantRes: &authtypes.Profile{ + Localpart: "alice", DisplayName: aliceDisplayName, + AvatarURL: aliceAvatarURL, + ServerName: string(serverName), }, }, { - req: api.QueryProfileRequest{ - UserID: fmt.Sprintf("@bob:%s", serverName), - }, - wantRes: api.QueryProfileResponse{ - UserExists: false, - }, + userID: fmt.Sprintf("@bob:%s", serverName), + wantErr: api2.ErrProfileNotExists, }, { - req: api.QueryProfileRequest{ - UserID: "@alice:wrongdomain.com", - }, - wantErr: fmt.Errorf("wrong domain"), + userID: "@alice:wrongdomain.com", + wantErr: api2.ErrProfileNotExists, }, } @@ -147,14 +142,14 @@ func TestQueryProfile(t *testing.T) { mode = "HTTP" } for _, tc := range testCases { - var gotRes api.QueryProfileResponse - gotErr := testAPI.QueryProfile(context.TODO(), &tc.req, &gotRes) + + profile, gotErr := testAPI.QueryProfile(context.TODO(), tc.userID) if tc.wantErr == nil && gotErr != nil || tc.wantErr != nil && gotErr == nil { t.Errorf("QueryProfile %s error, got %s want %s", mode, gotErr, tc.wantErr) continue } - if !reflect.DeepEqual(tc.wantRes, gotRes) { - t.Errorf("QueryProfile %s response got %+v want %+v", mode, gotRes, tc.wantRes) + if !reflect.DeepEqual(tc.wantRes, profile) { + t.Errorf("QueryProfile %s response got %+v want %+v", mode, profile, tc.wantRes) } } } From 560ba4627272b1ce5afc1f382871dd1967c836bc Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Mon, 3 Apr 2023 21:21:06 +0200 Subject: [PATCH 29/42] Add tests for CSAPI membership changes (#3034) Adds some more checks in regards to power levels, uses a less heavy way to get the membership of a user, avoids asking the database for the room version, since it will be queried later. [skip ci] --- clientapi/clientapi_test.go | 293 ++++++++++++++++++++++++++++++++ clientapi/routing/membership.go | 189 ++++++++++---------- clientapi/routing/sendtyping.go | 3 +- 3 files changed, 383 insertions(+), 102 deletions(-) diff --git a/clientapi/clientapi_test.go b/clientapi/clientapi_test.go index 2168a2b3a1..9ee827a823 100644 --- a/clientapi/clientapi_test.go +++ b/clientapi/clientapi_test.go @@ -22,6 +22,7 @@ import ( "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver" + "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/jetstream" "github.com/matrix-org/dendrite/test" @@ -600,3 +601,295 @@ func TestSetAvatarURL(t *testing.T) { } }) } + +func TestTyping(t *testing.T) { + alice := test.NewUser(t) + room := test.NewRoom(t, alice) + ctx := context.Background() + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + cfg, processCtx, close := testrig.CreateConfig(t, dbType) + defer close() + natsInstance := jetstream.NATSInstance{} + + routers := httputil.NewRouters() + cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) + caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) + rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) + rsAPI.SetFederationAPI(nil, nil) + // Needed to create accounts + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + // We mostly need the rsAPI/userAPI for this test, so nil for other APIs etc. + AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) + + // Create the users in the userapi and login + accessTokens := map[*test.User]userDevice{ + alice: {}, + } + createAccessTokens(t, accessTokens, userAPI, ctx, routers) + + // Create the room + if err := api.SendEvents(ctx, rsAPI, api.KindNew, room.Events(), "test", "test", "test", nil, false); err != nil { + t.Fatal(err) + } + + testCases := []struct { + name string + typingForUser string + roomID string + requestBody io.Reader + wantOK bool + }{ + { + name: "can not set typing for different user", + typingForUser: "@notourself:test", + roomID: room.ID, + requestBody: strings.NewReader(""), + }, + { + name: "invalid request body", + typingForUser: alice.ID, + roomID: room.ID, + requestBody: strings.NewReader(""), + }, + { + name: "non-existent room", + typingForUser: alice.ID, + roomID: "!doesnotexist:test", + }, + { + name: "invalid room ID", + typingForUser: alice.ID, + roomID: "@notaroomid:test", + }, + { + name: "allowed to set own typing status", + typingForUser: alice.ID, + roomID: room.ID, + requestBody: strings.NewReader(`{"typing":true}`), + wantOK: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPut, "/_matrix/client/v3/rooms/"+tc.roomID+"/typing/"+tc.typingForUser, tc.requestBody) + req.Header.Set("Authorization", "Bearer "+accessTokens[alice].accessToken) + routers.Client.ServeHTTP(rec, req) + if tc.wantOK && rec.Code != http.StatusOK { + t.Fatalf("expected HTTP 200, got %d: %s", rec.Code, rec.Body.String()) + } + }) + } + }) +} + +func TestMembership(t *testing.T) { + alice := test.NewUser(t) + bob := test.NewUser(t) + room := test.NewRoom(t, alice) + ctx := context.Background() + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + cfg, processCtx, close := testrig.CreateConfig(t, dbType) + cfg.ClientAPI.RateLimiting.Enabled = false + defer close() + natsInstance := jetstream.NATSInstance{} + + routers := httputil.NewRouters() + cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) + caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) + rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) + rsAPI.SetFederationAPI(nil, nil) + // Needed to create accounts + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + rsAPI.SetUserAPI(userAPI) + // We mostly need the rsAPI/userAPI for this test, so nil for other APIs etc. + AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) + + // Create the users in the userapi and login + accessTokens := map[*test.User]userDevice{ + alice: {}, + bob: {}, + } + createAccessTokens(t, accessTokens, userAPI, ctx, routers) + + // Create the room + if err := api.SendEvents(ctx, rsAPI, api.KindNew, room.Events(), "test", "test", "test", nil, false); err != nil { + t.Fatal(err) + } + + invalidBodyRequest := func(roomID, membershipType string) *http.Request { + return httptest.NewRequest(http.MethodPost, fmt.Sprintf("/_matrix/client/v3/rooms/%s/%s", roomID, membershipType), strings.NewReader("")) + } + + missingUserIDRequest := func(roomID, membershipType string) *http.Request { + return httptest.NewRequest(http.MethodPost, fmt.Sprintf("/_matrix/client/v3/rooms/%s/%s", roomID, membershipType), strings.NewReader("{}")) + } + + testCases := []struct { + name string + roomID string + request *http.Request + wantOK bool + asUser *test.User + }{ + { + name: "ban - invalid request body", + request: invalidBodyRequest(room.ID, "ban"), + }, + { + name: "kick - invalid request body", + request: invalidBodyRequest(room.ID, "kick"), + }, + { + name: "unban - invalid request body", + request: invalidBodyRequest(room.ID, "unban"), + }, + { + name: "invite - invalid request body", + request: invalidBodyRequest(room.ID, "invite"), + }, + { + name: "ban - missing user_id body", + request: missingUserIDRequest(room.ID, "ban"), + }, + { + name: "kick - missing user_id body", + request: missingUserIDRequest(room.ID, "kick"), + }, + { + name: "unban - missing user_id body", + request: missingUserIDRequest(room.ID, "unban"), + }, + { + name: "invite - missing user_id body", + request: missingUserIDRequest(room.ID, "invite"), + }, + { + name: "Bob forgets invalid room", + request: httptest.NewRequest(http.MethodPost, fmt.Sprintf("/_matrix/client/v3/rooms/%s/%s", "!doesnotexist", "forget"), strings.NewReader("")), + asUser: bob, + }, + { + name: "Alice can not ban Bob in non-existent room", // fails because "not joined" + request: httptest.NewRequest(http.MethodPost, fmt.Sprintf("/_matrix/client/v3/rooms/%s/%s", "!doesnotexist:test", "ban"), strings.NewReader(fmt.Sprintf(`{"user_id":"%s"}`, bob.ID))), + }, + { + name: "Alice can not kick Bob in non-existent room", // fails because "not joined" + request: httptest.NewRequest(http.MethodPost, fmt.Sprintf("/_matrix/client/v3/rooms/%s/%s", "!doesnotexist:test", "kick"), strings.NewReader(fmt.Sprintf(`{"user_id":"%s"}`, bob.ID))), + }, + // the following must run in sequence, as they build up on each other + { + name: "Alice invites Bob", + request: httptest.NewRequest(http.MethodPost, fmt.Sprintf("/_matrix/client/v3/rooms/%s/%s", room.ID, "invite"), strings.NewReader(fmt.Sprintf(`{"user_id":"%s"}`, bob.ID))), + wantOK: true, + }, + { + name: "Bob accepts invite", + request: httptest.NewRequest(http.MethodPost, fmt.Sprintf("/_matrix/client/v3/rooms/%s/%s", room.ID, "join"), strings.NewReader("")), + wantOK: true, + asUser: bob, + }, + { + name: "Alice verifies that Bob is joined", // returns an error if no membership event can be found + request: httptest.NewRequest(http.MethodGet, fmt.Sprintf("/_matrix/client/v3/rooms/%s/%s/m.room.member/%s", room.ID, "state", bob.ID), strings.NewReader("")), + wantOK: true, + }, + { + name: "Bob forgets the room but is still a member", + request: httptest.NewRequest(http.MethodPost, fmt.Sprintf("/_matrix/client/v3/rooms/%s/%s", room.ID, "forget"), strings.NewReader("")), + wantOK: false, // user is still in the room + asUser: bob, + }, + { + name: "Bob can not kick Alice", + request: httptest.NewRequest(http.MethodPost, fmt.Sprintf("/_matrix/client/v3/rooms/%s/%s", room.ID, "kick"), strings.NewReader(fmt.Sprintf(`{"user_id":"%s"}`, alice.ID))), + wantOK: false, // powerlevel too low + asUser: bob, + }, + { + name: "Bob can not ban Alice", + request: httptest.NewRequest(http.MethodPost, fmt.Sprintf("/_matrix/client/v3/rooms/%s/%s", room.ID, "ban"), strings.NewReader(fmt.Sprintf(`{"user_id":"%s"}`, alice.ID))), + wantOK: false, // powerlevel too low + asUser: bob, + }, + { + name: "Alice can kick Bob", + request: httptest.NewRequest(http.MethodPost, fmt.Sprintf("/_matrix/client/v3/rooms/%s/%s", room.ID, "kick"), strings.NewReader(fmt.Sprintf(`{"user_id":"%s"}`, bob.ID))), + wantOK: true, + }, + { + name: "Alice can ban Bob", + request: httptest.NewRequest(http.MethodPost, fmt.Sprintf("/_matrix/client/v3/rooms/%s/%s", room.ID, "ban"), strings.NewReader(fmt.Sprintf(`{"user_id":"%s"}`, bob.ID))), + wantOK: true, + }, + { + name: "Alice can not kick Bob again", + request: httptest.NewRequest(http.MethodPost, fmt.Sprintf("/_matrix/client/v3/rooms/%s/%s", room.ID, "kick"), strings.NewReader(fmt.Sprintf(`{"user_id":"%s"}`, bob.ID))), + wantOK: false, // can not kick banned/left user + }, + { + name: "Bob can not unban himself", // mostly because of not being a member of the room + request: httptest.NewRequest(http.MethodPost, fmt.Sprintf("/_matrix/client/v3/rooms/%s/%s", room.ID, "unban"), strings.NewReader(fmt.Sprintf(`{"user_id":"%s"}`, bob.ID))), + asUser: bob, + }, + { + name: "Alice can not invite Bob again", + request: httptest.NewRequest(http.MethodPost, fmt.Sprintf("/_matrix/client/v3/rooms/%s/%s", room.ID, "invite"), strings.NewReader(fmt.Sprintf(`{"user_id":"%s"}`, bob.ID))), + wantOK: false, // user still banned + }, + { + name: "Alice can unban Bob", + request: httptest.NewRequest(http.MethodPost, fmt.Sprintf("/_matrix/client/v3/rooms/%s/%s", room.ID, "unban"), strings.NewReader(fmt.Sprintf(`{"user_id":"%s"}`, bob.ID))), + wantOK: true, + }, + { + name: "Alice can not unban Bob again", + request: httptest.NewRequest(http.MethodPost, fmt.Sprintf("/_matrix/client/v3/rooms/%s/%s", room.ID, "unban"), strings.NewReader(fmt.Sprintf(`{"user_id":"%s"}`, bob.ID))), + wantOK: false, + }, + { + name: "Alice can invite Bob again", + request: httptest.NewRequest(http.MethodPost, fmt.Sprintf("/_matrix/client/v3/rooms/%s/%s", room.ID, "invite"), strings.NewReader(fmt.Sprintf(`{"user_id":"%s"}`, bob.ID))), + wantOK: true, + }, + { + name: "Bob can reject the invite by leaving", + request: httptest.NewRequest(http.MethodPost, fmt.Sprintf("/_matrix/client/v3/rooms/%s/%s", room.ID, "leave"), strings.NewReader("")), + wantOK: true, + asUser: bob, + }, + { + name: "Bob can forget the room", + request: httptest.NewRequest(http.MethodPost, fmt.Sprintf("/_matrix/client/v3/rooms/%s/%s", room.ID, "forget"), strings.NewReader("")), + wantOK: true, + asUser: bob, + }, + { + name: "Bob can forget the room again", + request: httptest.NewRequest(http.MethodPost, fmt.Sprintf("/_matrix/client/v3/rooms/%s/%s", room.ID, "forget"), strings.NewReader("")), + wantOK: true, + asUser: bob, + }, + // END must run in sequence + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + if tc.asUser == nil { + tc.asUser = alice + } + rec := httptest.NewRecorder() + tc.request.Header.Set("Authorization", "Bearer "+accessTokens[tc.asUser].accessToken) + routers.Client.ServeHTTP(rec, tc.request) + if tc.wantOK && rec.Code != http.StatusOK { + t.Fatalf("expected HTTP 200, got %d: %s", rec.Code, rec.Body.String()) + } + if !tc.wantOK && rec.Code == http.StatusOK { + t.Fatalf("expected request to fail, but didn't: %s", rec.Body.String()) + } + t.Logf("%s", rec.Body.String()) + }) + } + }) +} diff --git a/clientapi/routing/membership.go b/clientapi/routing/membership.go index 482c1f5f78..1a96d4b1dd 100644 --- a/clientapi/routing/membership.go +++ b/clientapi/routing/membership.go @@ -16,11 +16,12 @@ package routing import ( "context" - "errors" "fmt" "net/http" "time" + "github.com/matrix-org/gomatrixserverlib" + appserviceAPI "github.com/matrix-org/dendrite/appservice/api" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/httputil" @@ -31,76 +32,56 @@ import ( roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/setup/config" userapi "github.com/matrix-org/dendrite/userapi/api" - "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" ) -var errMissingUserID = errors.New("'user_id' must be supplied") - func SendBan( req *http.Request, profileAPI userapi.ClientUserAPI, device *userapi.Device, roomID string, cfg *config.ClientAPI, rsAPI roomserverAPI.ClientRoomserverAPI, asAPI appserviceAPI.AppServiceInternalAPI, ) util.JSONResponse { - body, evTime, roomVer, reqErr := extractRequestData(req, roomID, rsAPI) + body, evTime, reqErr := extractRequestData(req) if reqErr != nil { return *reqErr } + if body.UserID == "" { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.BadJSON("missing user_id"), + } + } + errRes := checkMemberInRoom(req.Context(), rsAPI, device.UserID, roomID) if errRes != nil { return *errRes } - plEvent := roomserverAPI.GetStateEvent(req.Context(), rsAPI, roomID, gomatrixserverlib.StateKeyTuple{ - EventType: gomatrixserverlib.MRoomPowerLevels, - StateKey: "", - }) - if plEvent == nil { - return util.JSONResponse{ - Code: 403, - JSON: jsonerror.Forbidden("You don't have permission to ban this user, no power_levels event in this room."), - } - } - pl, err := plEvent.PowerLevels() - if err != nil { - return util.JSONResponse{ - Code: 403, - JSON: jsonerror.Forbidden("You don't have permission to ban this user, the power_levels event for this room is malformed so auth checks cannot be performed."), - } + pl, errRes := getPowerlevels(req, rsAPI, roomID) + if errRes != nil { + return *errRes } allowedToBan := pl.UserLevel(device.UserID) >= pl.Ban if !allowedToBan { return util.JSONResponse{ - Code: 403, + Code: http.StatusForbidden, JSON: jsonerror.Forbidden("You don't have permission to ban this user, power level too low."), } } - return sendMembership(req.Context(), profileAPI, device, roomID, "ban", body.Reason, cfg, body.UserID, evTime, roomVer, rsAPI, asAPI) + return sendMembership(req.Context(), profileAPI, device, roomID, gomatrixserverlib.Ban, body.Reason, cfg, body.UserID, evTime, rsAPI, asAPI) } func sendMembership(ctx context.Context, profileAPI userapi.ClientUserAPI, device *userapi.Device, roomID, membership, reason string, cfg *config.ClientAPI, targetUserID string, evTime time.Time, - roomVer gomatrixserverlib.RoomVersion, rsAPI roomserverAPI.ClientRoomserverAPI, asAPI appserviceAPI.AppServiceInternalAPI) util.JSONResponse { event, err := buildMembershipEvent( ctx, targetUserID, reason, profileAPI, device, membership, roomID, false, cfg, evTime, rsAPI, asAPI, ) - if err == errMissingUserID { - return util.JSONResponse{ - Code: http.StatusBadRequest, - JSON: jsonerror.BadJSON(err.Error()), - } - } else if err == eventutil.ErrRoomNoExists { - return util.JSONResponse{ - Code: http.StatusNotFound, - JSON: jsonerror.NotFound(err.Error()), - } - } else if err != nil { + if err != nil { util.GetLogger(ctx).WithError(err).Error("buildMembershipEvent failed") return jsonerror.InternalServerError() } @@ -109,7 +90,7 @@ func sendMembership(ctx context.Context, profileAPI userapi.ClientUserAPI, devic if err = roomserverAPI.SendEvents( ctx, rsAPI, roomserverAPI.KindNew, - []*gomatrixserverlib.HeaderedEvent{event.Event.Headered(roomVer)}, + []*gomatrixserverlib.HeaderedEvent{event}, device.UserDomain(), serverName, serverName, @@ -131,13 +112,13 @@ func SendKick( roomID string, cfg *config.ClientAPI, rsAPI roomserverAPI.ClientRoomserverAPI, asAPI appserviceAPI.AppServiceInternalAPI, ) util.JSONResponse { - body, evTime, roomVer, reqErr := extractRequestData(req, roomID, rsAPI) + body, evTime, reqErr := extractRequestData(req) if reqErr != nil { return *reqErr } if body.UserID == "" { return util.JSONResponse{ - Code: 400, + Code: http.StatusBadRequest, JSON: jsonerror.BadJSON("missing user_id"), } } @@ -147,6 +128,18 @@ func SendKick( return *errRes } + pl, errRes := getPowerlevels(req, rsAPI, roomID) + if errRes != nil { + return *errRes + } + allowedToKick := pl.UserLevel(device.UserID) >= pl.Kick + if !allowedToKick { + return util.JSONResponse{ + Code: http.StatusForbidden, + JSON: jsonerror.Forbidden("You don't have permission to kick this user, power level too low."), + } + } + var queryRes roomserverAPI.QueryMembershipForUserResponse err := rsAPI.QueryMembershipForUser(req.Context(), &roomserverAPI.QueryMembershipForUserRequest{ RoomID: roomID, @@ -156,14 +149,14 @@ func SendKick( return util.ErrorResponse(err) } // kick is only valid if the user is not currently banned or left (that is, they are joined or invited) - if queryRes.Membership != "join" && queryRes.Membership != "invite" { + if queryRes.Membership != gomatrixserverlib.Join && queryRes.Membership != gomatrixserverlib.Invite { return util.JSONResponse{ - Code: 403, + Code: http.StatusForbidden, JSON: jsonerror.Unknown("cannot /kick banned or left users"), } } // TODO: should we be using SendLeave instead? - return sendMembership(req.Context(), profileAPI, device, roomID, "leave", body.Reason, cfg, body.UserID, evTime, roomVer, rsAPI, asAPI) + return sendMembership(req.Context(), profileAPI, device, roomID, gomatrixserverlib.Leave, body.Reason, cfg, body.UserID, evTime, rsAPI, asAPI) } func SendUnban( @@ -171,17 +164,22 @@ func SendUnban( roomID string, cfg *config.ClientAPI, rsAPI roomserverAPI.ClientRoomserverAPI, asAPI appserviceAPI.AppServiceInternalAPI, ) util.JSONResponse { - body, evTime, roomVer, reqErr := extractRequestData(req, roomID, rsAPI) + body, evTime, reqErr := extractRequestData(req) if reqErr != nil { return *reqErr } if body.UserID == "" { return util.JSONResponse{ - Code: 400, + Code: http.StatusBadRequest, JSON: jsonerror.BadJSON("missing user_id"), } } + errRes := checkMemberInRoom(req.Context(), rsAPI, device.UserID, roomID) + if errRes != nil { + return *errRes + } + var queryRes roomserverAPI.QueryMembershipForUserResponse err := rsAPI.QueryMembershipForUser(req.Context(), &roomserverAPI.QueryMembershipForUserRequest{ RoomID: roomID, @@ -190,21 +188,16 @@ func SendUnban( if err != nil { return util.ErrorResponse(err) } - if !queryRes.RoomExists { - return util.JSONResponse{ - Code: http.StatusForbidden, - JSON: jsonerror.Forbidden("room does not exist"), - } - } + // unban is only valid if the user is currently banned - if queryRes.Membership != "ban" { + if queryRes.Membership != gomatrixserverlib.Ban { return util.JSONResponse{ - Code: 400, + Code: http.StatusBadRequest, JSON: jsonerror.Unknown("can only /unban users that are banned"), } } // TODO: should we be using SendLeave instead? - return sendMembership(req.Context(), profileAPI, device, roomID, "leave", body.Reason, cfg, body.UserID, evTime, roomVer, rsAPI, asAPI) + return sendMembership(req.Context(), profileAPI, device, roomID, gomatrixserverlib.Leave, body.Reason, cfg, body.UserID, evTime, rsAPI, asAPI) } func SendInvite( @@ -212,7 +205,7 @@ func SendInvite( roomID string, cfg *config.ClientAPI, rsAPI roomserverAPI.ClientRoomserverAPI, asAPI appserviceAPI.AppServiceInternalAPI, ) util.JSONResponse { - body, evTime, _, reqErr := extractRequestData(req, roomID, rsAPI) + body, evTime, reqErr := extractRequestData(req) if reqErr != nil { return *reqErr } @@ -234,6 +227,18 @@ func SendInvite( } } + if body.UserID == "" { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.BadJSON("missing user_id"), + } + } + + errRes := checkMemberInRoom(req.Context(), rsAPI, device.UserID, roomID) + if errRes != nil { + return *errRes + } + // We already received the return value, so no need to check for an error here. response, _ := sendInvite(req.Context(), profileAPI, device, roomID, body.UserID, body.Reason, cfg, rsAPI, asAPI, evTime) return response @@ -250,20 +255,10 @@ func sendInvite( asAPI appserviceAPI.AppServiceInternalAPI, evTime time.Time, ) (util.JSONResponse, error) { event, err := buildMembershipEvent( - ctx, userID, reason, profileAPI, device, "invite", + ctx, userID, reason, profileAPI, device, gomatrixserverlib.Invite, roomID, false, cfg, evTime, rsAPI, asAPI, ) - if err == errMissingUserID { - return util.JSONResponse{ - Code: http.StatusBadRequest, - JSON: jsonerror.BadJSON(err.Error()), - }, err - } else if err == eventutil.ErrRoomNoExists { - return util.JSONResponse{ - Code: http.StatusNotFound, - JSON: jsonerror.NotFound(err.Error()), - }, err - } else if err != nil { + if err != nil { util.GetLogger(ctx).WithError(err).Error("buildMembershipEvent failed") return jsonerror.InternalServerError(), err } @@ -357,19 +352,7 @@ func loadProfile( return profile, err } -func extractRequestData(req *http.Request, roomID string, rsAPI roomserverAPI.ClientRoomserverAPI) ( - body *threepid.MembershipRequest, evTime time.Time, roomVer gomatrixserverlib.RoomVersion, resErr *util.JSONResponse, -) { - verReq := roomserverAPI.QueryRoomVersionForRoomRequest{RoomID: roomID} - verRes := roomserverAPI.QueryRoomVersionForRoomResponse{} - if err := rsAPI.QueryRoomVersionForRoom(req.Context(), &verReq, &verRes); err != nil { - resErr = &util.JSONResponse{ - Code: http.StatusBadRequest, - JSON: jsonerror.UnsupportedRoomVersion(err.Error()), - } - return - } - roomVer = verRes.RoomVersion +func extractRequestData(req *http.Request) (body *threepid.MembershipRequest, evTime time.Time, resErr *util.JSONResponse) { if reqErr := httputil.UnmarshalJSONRequest(req, &body); reqErr != nil { resErr = reqErr @@ -432,34 +415,17 @@ func checkAndProcessThreepid( } func checkMemberInRoom(ctx context.Context, rsAPI roomserverAPI.ClientRoomserverAPI, userID, roomID string) *util.JSONResponse { - tuple := gomatrixserverlib.StateKeyTuple{ - EventType: gomatrixserverlib.MRoomMember, - StateKey: userID, - } - var membershipRes roomserverAPI.QueryCurrentStateResponse - err := rsAPI.QueryCurrentState(ctx, &roomserverAPI.QueryCurrentStateRequest{ - RoomID: roomID, - StateTuples: []gomatrixserverlib.StateKeyTuple{tuple}, + var membershipRes roomserverAPI.QueryMembershipForUserResponse + err := rsAPI.QueryMembershipForUser(ctx, &roomserverAPI.QueryMembershipForUserRequest{ + RoomID: roomID, + UserID: userID, }, &membershipRes) if err != nil { - util.GetLogger(ctx).WithError(err).Error("QueryCurrentState: could not query membership for user") - e := jsonerror.InternalServerError() - return &e - } - ev := membershipRes.StateEvents[tuple] - if ev == nil { - return &util.JSONResponse{ - Code: http.StatusForbidden, - JSON: jsonerror.Forbidden("user does not belong to room"), - } - } - membership, err := ev.Membership() - if err != nil { - util.GetLogger(ctx).WithError(err).Error("Member event isn't valid") + util.GetLogger(ctx).WithError(err).Error("QueryMembershipForUser: could not query membership for user") e := jsonerror.InternalServerError() return &e } - if membership != gomatrixserverlib.Join { + if !membershipRes.IsInRoom { return &util.JSONResponse{ Code: http.StatusForbidden, JSON: jsonerror.Forbidden("user does not belong to room"), @@ -511,3 +477,24 @@ func SendForget( JSON: struct{}{}, } } + +func getPowerlevels(req *http.Request, rsAPI roomserverAPI.ClientRoomserverAPI, roomID string) (*gomatrixserverlib.PowerLevelContent, *util.JSONResponse) { + plEvent := roomserverAPI.GetStateEvent(req.Context(), rsAPI, roomID, gomatrixserverlib.StateKeyTuple{ + EventType: gomatrixserverlib.MRoomPowerLevels, + StateKey: "", + }) + if plEvent == nil { + return nil, &util.JSONResponse{ + Code: http.StatusForbidden, + JSON: jsonerror.Forbidden("You don't have permission to perform this action, no power_levels event in this room."), + } + } + pl, err := plEvent.PowerLevels() + if err != nil { + return nil, &util.JSONResponse{ + Code: http.StatusForbidden, + JSON: jsonerror.Forbidden("You don't have permission to perform this action, the power_levels event for this room is malformed so auth checks cannot be performed."), + } + } + return pl, nil +} diff --git a/clientapi/routing/sendtyping.go b/clientapi/routing/sendtyping.go index 3f92e42271..9dc884d627 100644 --- a/clientapi/routing/sendtyping.go +++ b/clientapi/routing/sendtyping.go @@ -15,12 +15,13 @@ package routing import ( "net/http" + "github.com/matrix-org/util" + "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/clientapi/producers" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" userapi "github.com/matrix-org/dendrite/userapi/api" - "github.com/matrix-org/util" ) type typingContentJSON struct { From 682a7d0a66ce0dfd34cff2899daa6f16fdc7ebae Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Mon, 3 Apr 2023 21:42:46 +0200 Subject: [PATCH 30/42] Add tests for `/turnServer`, `/capabilities` and `/3pid/` (#3038) Threepid seems to be pretty out of date, several missing endpoints. Should also fix #3037, where we were still listening on the `/unstable` prefix, while Element Web uses `/r0` --- clientapi/auth/authtypes/threepid.go | 6 +- clientapi/clientapi_test.go | 342 +++++++++++++++++++++++++++ clientapi/routing/capabilities.go | 30 ++- clientapi/routing/routing.go | 11 +- clientapi/routing/threepid.go | 26 +- clientapi/threepid/threepid.go | 41 ++-- roomserver/api/api.go | 1 - roomserver/api/query.go | 9 - roomserver/internal/query/query.go | 21 +- 9 files changed, 402 insertions(+), 85 deletions(-) diff --git a/clientapi/auth/authtypes/threepid.go b/clientapi/auth/authtypes/threepid.go index 60d77dc6f0..ebafbe6aa1 100644 --- a/clientapi/auth/authtypes/threepid.go +++ b/clientapi/auth/authtypes/threepid.go @@ -16,6 +16,8 @@ package authtypes // ThreePID represents a third-party identifier type ThreePID struct { - Address string `json:"address"` - Medium string `json:"medium"` + Address string `json:"address"` + Medium string `json:"medium"` + AddedAt int64 `json:"added_at"` + ValidatedAt int64 `json:"validated_at"` } diff --git a/clientapi/clientapi_test.go b/clientapi/clientapi_test.go index 9ee827a823..76295ba598 100644 --- a/clientapi/clientapi_test.go +++ b/clientapi/clientapi_test.go @@ -12,18 +12,24 @@ import ( "testing" "time" + "github.com/matrix-org/gomatrix" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" + "github.com/stretchr/testify/assert" "github.com/tidwall/gjson" "github.com/matrix-org/dendrite/appservice" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" + "github.com/matrix-org/dendrite/clientapi/routing" + "github.com/matrix-org/dendrite/clientapi/threepid" "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver" "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/roomserver/version" "github.com/matrix-org/dendrite/setup/base" + "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/jetstream" "github.com/matrix-org/dendrite/test" "github.com/matrix-org/dendrite/test/testrig" @@ -893,3 +899,339 @@ func TestMembership(t *testing.T) { } }) } + +func TestCapabilities(t *testing.T) { + alice := test.NewUser(t) + ctx := context.Background() + + // construct the expected result + versionsMap := map[gomatrixserverlib.RoomVersion]string{} + for v, desc := range version.SupportedRoomVersions() { + if desc.Stable { + versionsMap[v] = "stable" + } else { + versionsMap[v] = "unstable" + } + } + + expectedMap := map[string]interface{}{ + "capabilities": map[string]interface{}{ + "m.change_password": map[string]bool{ + "enabled": true, + }, + "m.room_versions": map[string]interface{}{ + "default": version.DefaultRoomVersion(), + "available": versionsMap, + }, + }, + } + + expectedBuf := &bytes.Buffer{} + err := json.NewEncoder(expectedBuf).Encode(expectedMap) + assert.NoError(t, err) + + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + cfg, processCtx, close := testrig.CreateConfig(t, dbType) + cfg.ClientAPI.RateLimiting.Enabled = false + defer close() + natsInstance := jetstream.NATSInstance{} + + routers := httputil.NewRouters() + cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) + + // Needed to create accounts + rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, caching.DisableMetrics) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + // We mostly need the rsAPI/userAPI for this test, so nil for other APIs etc. + AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) + + // Create the users in the userapi and login + accessTokens := map[*test.User]userDevice{ + alice: {}, + } + createAccessTokens(t, accessTokens, userAPI, ctx, routers) + + testCases := []struct { + name string + request *http.Request + }{ + { + name: "can get capabilities", + request: httptest.NewRequest(http.MethodGet, "/_matrix/client/v3/capabilities", strings.NewReader("")), + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + rec := httptest.NewRecorder() + tc.request.Header.Set("Authorization", "Bearer "+accessTokens[alice].accessToken) + routers.Client.ServeHTTP(rec, tc.request) + assert.Equal(t, http.StatusOK, rec.Code) + assert.ObjectsAreEqual(expectedBuf.Bytes(), rec.Body.Bytes()) + }) + } + }) +} + +func TestTurnserver(t *testing.T) { + alice := test.NewUser(t) + ctx := context.Background() + + cfg, processCtx, close := testrig.CreateConfig(t, test.DBTypeSQLite) + cfg.ClientAPI.RateLimiting.Enabled = false + defer close() + natsInstance := jetstream.NATSInstance{} + + routers := httputil.NewRouters() + cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) + + // Needed to create accounts + rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, caching.DisableMetrics) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + //rsAPI.SetUserAPI(userAPI) + // We mostly need the rsAPI/userAPI for this test, so nil for other APIs etc. + AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) + + // Create the users in the userapi and login + accessTokens := map[*test.User]userDevice{ + alice: {}, + } + createAccessTokens(t, accessTokens, userAPI, ctx, routers) + + testCases := []struct { + name string + turnConfig config.TURN + wantEmptyResponse bool + }{ + { + name: "no turn server configured", + wantEmptyResponse: true, + }, + { + name: "servers configured but not userLifeTime", + wantEmptyResponse: true, + turnConfig: config.TURN{URIs: []string{""}}, + }, + { + name: "missing sharedSecret/username/password", + wantEmptyResponse: true, + turnConfig: config.TURN{URIs: []string{""}, UserLifetime: "1m"}, + }, + { + name: "with shared secret", + turnConfig: config.TURN{URIs: []string{""}, UserLifetime: "1m", SharedSecret: "iAmSecret"}, + }, + { + name: "with username/password secret", + turnConfig: config.TURN{URIs: []string{""}, UserLifetime: "1m", Username: "username", Password: "iAmSecret"}, + }, + { + name: "only username set", + turnConfig: config.TURN{URIs: []string{""}, UserLifetime: "1m", Username: "username"}, + wantEmptyResponse: true, + }, + { + name: "only password set", + turnConfig: config.TURN{URIs: []string{""}, UserLifetime: "1m", Username: "username"}, + wantEmptyResponse: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/_matrix/client/v3/voip/turnServer", strings.NewReader("")) + req.Header.Set("Authorization", "Bearer "+accessTokens[alice].accessToken) + cfg.ClientAPI.TURN = tc.turnConfig + routers.Client.ServeHTTP(rec, req) + assert.Equal(t, http.StatusOK, rec.Code) + + if tc.wantEmptyResponse && rec.Body.String() != "{}" { + t.Fatalf("expected an empty response, but got %s", rec.Body.String()) + } + if !tc.wantEmptyResponse { + assert.NotEqual(t, "{}", rec.Body.String()) + + resp := gomatrix.RespTurnServer{} + err := json.NewDecoder(rec.Body).Decode(&resp) + assert.NoError(t, err) + + duration, _ := time.ParseDuration(tc.turnConfig.UserLifetime) + assert.Equal(t, tc.turnConfig.URIs, resp.URIs) + assert.Equal(t, int(duration.Seconds()), resp.TTL) + if tc.turnConfig.Username != "" && tc.turnConfig.Password != "" { + assert.Equal(t, tc.turnConfig.Username, resp.Username) + assert.Equal(t, tc.turnConfig.Password, resp.Password) + } + } + }) + } +} + +func Test3PID(t *testing.T) { + alice := test.NewUser(t) + ctx := context.Background() + + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + cfg, processCtx, close := testrig.CreateConfig(t, dbType) + cfg.ClientAPI.RateLimiting.Enabled = false + cfg.FederationAPI.DisableTLSValidation = true // needed to be able to connect to our identityServer below + defer close() + natsInstance := jetstream.NATSInstance{} + + routers := httputil.NewRouters() + cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) + + // Needed to create accounts + rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, caching.DisableMetrics) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + // We mostly need the rsAPI/userAPI for this test, so nil for other APIs etc. + AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) + + // Create the users in the userapi and login + accessTokens := map[*test.User]userDevice{ + alice: {}, + } + createAccessTokens(t, accessTokens, userAPI, ctx, routers) + + identityServer := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case strings.Contains(r.URL.String(), "getValidated3pid"): + resp := threepid.GetValidatedResponse{} + switch r.URL.Query().Get("client_secret") { + case "fail": + resp.ErrCode = "M_SESSION_NOT_VALIDATED" + case "fail2": + resp.ErrCode = "some other error" + case "fail3": + _, _ = w.Write([]byte("{invalidJson")) + return + case "success": + resp.Medium = "email" + case "success2": + resp.Medium = "email" + resp.Address = "somerandom@address.com" + } + _ = json.NewEncoder(w).Encode(resp) + case strings.Contains(r.URL.String(), "requestToken"): + resp := threepid.SID{SID: "randomSID"} + _ = json.NewEncoder(w).Encode(resp) + } + })) + defer identityServer.Close() + + identityServerBase := strings.TrimPrefix(identityServer.URL, "https://") + + testCases := []struct { + name string + request *http.Request + wantOK bool + setTrustedServer bool + wantLen3PIDs int + }{ + { + name: "can get associated threepid info", + request: httptest.NewRequest(http.MethodGet, "/_matrix/client/v3/account/3pid", strings.NewReader("")), + wantOK: true, + }, + { + name: "can not set threepid info with invalid JSON", + request: httptest.NewRequest(http.MethodPost, "/_matrix/client/v3/account/3pid", strings.NewReader("")), + }, + { + name: "can not set threepid info with untrusted server", + request: httptest.NewRequest(http.MethodPost, "/_matrix/client/v3/account/3pid", strings.NewReader("{}")), + }, + { + name: "can check threepid info with trusted server, but unverified", + request: httptest.NewRequest(http.MethodPost, "/_matrix/client/v3/account/3pid", strings.NewReader(fmt.Sprintf(`{"three_pid_creds":{"id_server":"%s","client_secret":"fail"}}`, identityServerBase))), + setTrustedServer: true, + wantOK: false, + }, + { + name: "can check threepid info with trusted server, but fails for some other reason", + request: httptest.NewRequest(http.MethodPost, "/_matrix/client/v3/account/3pid", strings.NewReader(fmt.Sprintf(`{"three_pid_creds":{"id_server":"%s","client_secret":"fail2"}}`, identityServerBase))), + setTrustedServer: true, + wantOK: false, + }, + { + name: "can check threepid info with trusted server, but fails because of invalid json", + request: httptest.NewRequest(http.MethodPost, "/_matrix/client/v3/account/3pid", strings.NewReader(fmt.Sprintf(`{"three_pid_creds":{"id_server":"%s","client_secret":"fail3"}}`, identityServerBase))), + setTrustedServer: true, + wantOK: false, + }, + { + name: "can save threepid info with trusted server", + request: httptest.NewRequest(http.MethodPost, "/_matrix/client/v3/account/3pid", strings.NewReader(fmt.Sprintf(`{"three_pid_creds":{"id_server":"%s","client_secret":"success"}}`, identityServerBase))), + setTrustedServer: true, + wantOK: true, + }, + { + name: "can save threepid info with trusted server using bind=true", + request: httptest.NewRequest(http.MethodPost, "/_matrix/client/v3/account/3pid", strings.NewReader(fmt.Sprintf(`{"three_pid_creds":{"id_server":"%s","client_secret":"success2"},"bind":true}`, identityServerBase))), + setTrustedServer: true, + wantOK: true, + }, + { + name: "can get associated threepid info again", + request: httptest.NewRequest(http.MethodGet, "/_matrix/client/v3/account/3pid", strings.NewReader("")), + wantOK: true, + wantLen3PIDs: 2, + }, + { + name: "can delete associated threepid info", + request: httptest.NewRequest(http.MethodPost, "/_matrix/client/v3/account/3pid/delete", strings.NewReader(`{"medium":"email","address":"somerandom@address.com"}`)), + wantOK: true, + }, + { + name: "can get associated threepid after deleting association", + request: httptest.NewRequest(http.MethodGet, "/_matrix/client/v3/account/3pid", strings.NewReader("")), + wantOK: true, + wantLen3PIDs: 1, + }, + { + name: "can not request emailToken with invalid request body", + request: httptest.NewRequest(http.MethodPost, "/_matrix/client/v3/account/3pid/email/requestToken", strings.NewReader("")), + }, + { + name: "can not request emailToken for in use address", + request: httptest.NewRequest(http.MethodPost, "/_matrix/client/v3/account/3pid/email/requestToken", strings.NewReader(fmt.Sprintf(`{"client_secret":"somesecret","email":"","send_attempt":1,"id_server":"%s"}`, identityServerBase))), + }, + { + name: "can request emailToken", + request: httptest.NewRequest(http.MethodPost, "/_matrix/client/v3/account/3pid/email/requestToken", strings.NewReader(fmt.Sprintf(`{"client_secret":"somesecret","email":"somerandom@address.com","send_attempt":1,"id_server":"%s"}`, identityServerBase))), + wantOK: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + + if tc.setTrustedServer { + cfg.Global.TrustedIDServers = []string{identityServerBase} + } + + rec := httptest.NewRecorder() + tc.request.Header.Set("Authorization", "Bearer "+accessTokens[alice].accessToken) + + routers.Client.ServeHTTP(rec, tc.request) + t.Logf("Response: %s", rec.Body.String()) + if tc.wantOK && rec.Code != http.StatusOK { + t.Fatalf("expected HTTP 200, got %d: %s", rec.Code, rec.Body.String()) + } + if !tc.wantOK && rec.Code == http.StatusOK { + t.Fatalf("expected request to fail, but didn't: %s", rec.Body.String()) + } + if tc.wantLen3PIDs > 0 { + var resp routing.ThreePIDsResponse + if err := json.NewDecoder(rec.Body).Decode(&resp); err != nil { + t.Fatal(err) + } + if len(resp.ThreePIDs) != tc.wantLen3PIDs { + t.Fatalf("expected %d threepids, got %d", tc.wantLen3PIDs, len(resp.ThreePIDs)) + } + } + }) + } + }) +} diff --git a/clientapi/routing/capabilities.go b/clientapi/routing/capabilities.go index b7d47e916e..e6c1a9b8cb 100644 --- a/clientapi/routing/capabilities.go +++ b/clientapi/routing/capabilities.go @@ -17,26 +17,21 @@ package routing import ( "net/http" - "github.com/matrix-org/dendrite/clientapi/jsonerror" - roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" - + "github.com/matrix-org/dendrite/roomserver/version" + "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" ) // GetCapabilities returns information about the server's supported feature set // and other relevant capabilities to an authenticated user. -func GetCapabilities( - req *http.Request, rsAPI roomserverAPI.ClientRoomserverAPI, -) util.JSONResponse { - roomVersionsQueryReq := roomserverAPI.QueryRoomVersionCapabilitiesRequest{} - roomVersionsQueryRes := roomserverAPI.QueryRoomVersionCapabilitiesResponse{} - if err := rsAPI.QueryRoomVersionCapabilities( - req.Context(), - &roomVersionsQueryReq, - &roomVersionsQueryRes, - ); err != nil { - util.GetLogger(req.Context()).WithError(err).Error("queryAPI.QueryRoomVersionCapabilities failed") - return jsonerror.InternalServerError() +func GetCapabilities() util.JSONResponse { + versionsMap := map[gomatrixserverlib.RoomVersion]string{} + for v, desc := range version.SupportedRoomVersions() { + if desc.Stable { + versionsMap[v] = "stable" + } else { + versionsMap[v] = "unstable" + } } response := map[string]interface{}{ @@ -44,7 +39,10 @@ func GetCapabilities( "m.change_password": map[string]bool{ "enabled": true, }, - "m.room_versions": roomVersionsQueryRes, + "m.room_versions": map[string]interface{}{ + "default": version.DefaultRoomVersion(), + "available": versionsMap, + }, }, } diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index 58bea4ac70..246aa9e7fe 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -20,6 +20,7 @@ import ( "strings" "github.com/gorilla/mux" + "github.com/matrix-org/dendrite/setup/base" userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" @@ -849,6 +850,8 @@ func Setup( // Browsers use the OPTIONS HTTP method to check if the CORS policy allows // PUT requests, so we need to allow this method + threePIDClient := base.CreateClient(dendriteCfg, nil) // TODO: Move this somewhere else, e.g. pass in as parameter + v3mux.Handle("/account/3pid", httputil.MakeAuthAPI("account_3pid", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { return GetAssociated3PIDs(req, userAPI, device) @@ -857,11 +860,11 @@ func Setup( v3mux.Handle("/account/3pid", httputil.MakeAuthAPI("account_3pid", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - return CheckAndSave3PIDAssociation(req, userAPI, device, cfg) + return CheckAndSave3PIDAssociation(req, userAPI, device, cfg, threePIDClient) }), ).Methods(http.MethodPost, http.MethodOptions) - unstableMux.Handle("/account/3pid/delete", + v3mux.Handle("/account/3pid/delete", httputil.MakeAuthAPI("account_3pid", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { return Forget3PID(req, userAPI) }), @@ -869,7 +872,7 @@ func Setup( v3mux.Handle("/{path:(?:account/3pid|register)}/email/requestToken", httputil.MakeExternalAPI("account_3pid_request_token", func(req *http.Request) util.JSONResponse { - return RequestEmailToken(req, userAPI, cfg) + return RequestEmailToken(req, userAPI, cfg, threePIDClient) }), ).Methods(http.MethodPost, http.MethodOptions) @@ -1182,7 +1185,7 @@ func Setup( if r := rateLimits.Limit(req, device); r != nil { return *r } - return GetCapabilities(req, rsAPI) + return GetCapabilities() }, httputil.WithAllowGuests()), ).Methods(http.MethodGet, http.MethodOptions) diff --git a/clientapi/routing/threepid.go b/clientapi/routing/threepid.go index 971bfcad3f..8b3548c1fc 100644 --- a/clientapi/routing/threepid.go +++ b/clientapi/routing/threepid.go @@ -33,7 +33,7 @@ type reqTokenResponse struct { SID string `json:"sid"` } -type threePIDsResponse struct { +type ThreePIDsResponse struct { ThreePIDs []authtypes.ThreePID `json:"threepids"` } @@ -41,7 +41,7 @@ type threePIDsResponse struct { // // POST /account/3pid/email/requestToken // POST /register/email/requestToken -func RequestEmailToken(req *http.Request, threePIDAPI api.ClientUserAPI, cfg *config.ClientAPI) util.JSONResponse { +func RequestEmailToken(req *http.Request, threePIDAPI api.ClientUserAPI, cfg *config.ClientAPI, client *gomatrixserverlib.Client) util.JSONResponse { var body threepid.EmailAssociationRequest if reqErr := httputil.UnmarshalJSONRequest(req, &body); reqErr != nil { return *reqErr @@ -72,7 +72,7 @@ func RequestEmailToken(req *http.Request, threePIDAPI api.ClientUserAPI, cfg *co } } - resp.SID, err = threepid.CreateSession(req.Context(), body, cfg) + resp.SID, err = threepid.CreateSession(req.Context(), body, cfg, client) if err == threepid.ErrNotTrusted { return util.JSONResponse{ Code: http.StatusBadRequest, @@ -92,7 +92,7 @@ func RequestEmailToken(req *http.Request, threePIDAPI api.ClientUserAPI, cfg *co // CheckAndSave3PIDAssociation implements POST /account/3pid func CheckAndSave3PIDAssociation( req *http.Request, threePIDAPI api.ClientUserAPI, device *api.Device, - cfg *config.ClientAPI, + cfg *config.ClientAPI, client *gomatrixserverlib.Client, ) util.JSONResponse { var body threepid.EmailAssociationCheckRequest if reqErr := httputil.UnmarshalJSONRequest(req, &body); reqErr != nil { @@ -100,7 +100,7 @@ func CheckAndSave3PIDAssociation( } // Check if the association has been validated - verified, address, medium, err := threepid.CheckAssociation(req.Context(), body.Creds, cfg) + verified, address, medium, err := threepid.CheckAssociation(req.Context(), body.Creds, cfg, client) if err == threepid.ErrNotTrusted { return util.JSONResponse{ Code: http.StatusBadRequest, @@ -123,13 +123,8 @@ func CheckAndSave3PIDAssociation( if body.Bind { // Publish the association on the identity server if requested - err = threepid.PublishAssociation(body.Creds, device.UserID, cfg) - if err == threepid.ErrNotTrusted { - return util.JSONResponse{ - Code: http.StatusBadRequest, - JSON: jsonerror.NotTrusted(body.Creds.IDServer), - } - } else if err != nil { + err = threepid.PublishAssociation(req.Context(), body.Creds, device.UserID, cfg, client) + if err != nil { util.GetLogger(req.Context()).WithError(err).Error("threepid.PublishAssociation failed") return jsonerror.InternalServerError() } @@ -180,7 +175,7 @@ func GetAssociated3PIDs( return util.JSONResponse{ Code: http.StatusOK, - JSON: threePIDsResponse{res.ThreePIDs}, + JSON: ThreePIDsResponse{res.ThreePIDs}, } } @@ -191,7 +186,10 @@ func Forget3PID(req *http.Request, threepidAPI api.ClientUserAPI) util.JSONRespo return *reqErr } - if err := threepidAPI.PerformForgetThreePID(req.Context(), &api.PerformForgetThreePIDRequest{}, &struct{}{}); err != nil { + if err := threepidAPI.PerformForgetThreePID(req.Context(), &api.PerformForgetThreePIDRequest{ + ThreePID: body.Address, + Medium: body.Medium, + }, &struct{}{}); err != nil { util.GetLogger(req.Context()).WithError(err).Error("threepidAPI.PerformForgetThreePID failed") return jsonerror.InternalServerError() } diff --git a/clientapi/threepid/threepid.go b/clientapi/threepid/threepid.go index 1e64e30349..ac4dc38187 100644 --- a/clientapi/threepid/threepid.go +++ b/clientapi/threepid/threepid.go @@ -25,6 +25,7 @@ import ( "strings" "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/gomatrixserverlib" ) // EmailAssociationRequest represents the request defined at https://matrix.org/docs/spec/client_server/r0.2.0.html#post-matrix-client-r0-register-email-requesttoken @@ -37,7 +38,7 @@ type EmailAssociationRequest struct { // EmailAssociationCheckRequest represents the request defined at https://matrix.org/docs/spec/client_server/r0.2.0.html#post-matrix-client-r0-account-3pid type EmailAssociationCheckRequest struct { - Creds Credentials `json:"threePidCreds"` + Creds Credentials `json:"three_pid_creds"` Bind bool `json:"bind"` } @@ -48,12 +49,16 @@ type Credentials struct { Secret string `json:"client_secret"` } +type SID struct { + SID string `json:"sid"` +} + // CreateSession creates a session on an identity server. // Returns the session's ID. // Returns an error if there was a problem sending the request or decoding the // response, or if the identity server responded with a non-OK status. func CreateSession( - ctx context.Context, req EmailAssociationRequest, cfg *config.ClientAPI, + ctx context.Context, req EmailAssociationRequest, cfg *config.ClientAPI, client *gomatrixserverlib.Client, ) (string, error) { if err := isTrusted(req.IDServer, cfg); err != nil { return "", err @@ -73,8 +78,7 @@ func CreateSession( } request.Header.Add("Content-Type", "application/x-www-form-urlencoded") - client := http.Client{} - resp, err := client.Do(request.WithContext(ctx)) + resp, err := client.DoHTTPRequest(ctx, request) if err != nil { return "", err } @@ -85,14 +89,20 @@ func CreateSession( } // Extract the SID from the response and return it - var sid struct { - SID string `json:"sid"` - } + var sid SID err = json.NewDecoder(resp.Body).Decode(&sid) return sid.SID, err } +type GetValidatedResponse struct { + Medium string `json:"medium"` + ValidatedAt int64 `json:"validated_at"` + Address string `json:"address"` + ErrCode string `json:"errcode"` + Error string `json:"error"` +} + // CheckAssociation checks the status of an ongoing association validation on an // identity server. // Returns a boolean set to true if the association has been validated, false if not. @@ -102,6 +112,7 @@ func CreateSession( // response, or if the identity server responded with a non-OK status. func CheckAssociation( ctx context.Context, creds Credentials, cfg *config.ClientAPI, + client *gomatrixserverlib.Client, ) (bool, string, string, error) { if err := isTrusted(creds.IDServer, cfg); err != nil { return false, "", "", err @@ -112,19 +123,12 @@ func CheckAssociation( if err != nil { return false, "", "", err } - resp, err := http.DefaultClient.Do(req.WithContext(ctx)) + resp, err := client.DoHTTPRequest(ctx, req) if err != nil { return false, "", "", err } - var respBody struct { - Medium string `json:"medium"` - ValidatedAt int64 `json:"validated_at"` - Address string `json:"address"` - ErrCode string `json:"errcode"` - Error string `json:"error"` - } - + var respBody GetValidatedResponse if err = json.NewDecoder(resp.Body).Decode(&respBody); err != nil { return false, "", "", err } @@ -142,7 +146,7 @@ func CheckAssociation( // identifier and a Matrix ID. // Returns an error if there was a problem sending the request or decoding the // response, or if the identity server responded with a non-OK status. -func PublishAssociation(creds Credentials, userID string, cfg *config.ClientAPI) error { +func PublishAssociation(ctx context.Context, creds Credentials, userID string, cfg *config.ClientAPI, client *gomatrixserverlib.Client) error { if err := isTrusted(creds.IDServer, cfg); err != nil { return err } @@ -160,8 +164,7 @@ func PublishAssociation(creds Credentials, userID string, cfg *config.ClientAPI) } request.Header.Add("Content-Type", "application/x-www-form-urlencoded") - client := http.Client{} - resp, err := client.Do(request) + resp, err := client.DoHTTPRequest(ctx, request) if err != nil { return err } diff --git a/roomserver/api/api.go b/roomserver/api/api.go index f6d003a447..dda5bb5a4b 100644 --- a/roomserver/api/api.go +++ b/roomserver/api/api.go @@ -144,7 +144,6 @@ type ClientRoomserverAPI interface { QueryKnownUsers(ctx context.Context, req *QueryKnownUsersRequest, res *QueryKnownUsersResponse) error QueryRoomVersionForRoom(ctx context.Context, req *QueryRoomVersionForRoomRequest, res *QueryRoomVersionForRoomResponse) error QueryPublishedRooms(ctx context.Context, req *QueryPublishedRoomsRequest, res *QueryPublishedRoomsResponse) error - QueryRoomVersionCapabilities(ctx context.Context, req *QueryRoomVersionCapabilitiesRequest, res *QueryRoomVersionCapabilitiesResponse) error GetRoomIDForAlias(ctx context.Context, req *GetRoomIDForAliasRequest, res *GetRoomIDForAliasResponse) error GetAliasesForRoomID(ctx context.Context, req *GetAliasesForRoomIDRequest, res *GetAliasesForRoomIDResponse) error diff --git a/roomserver/api/query.go b/roomserver/api/query.go index 24722db0ba..d60ec4f60e 100644 --- a/roomserver/api/query.go +++ b/roomserver/api/query.go @@ -240,15 +240,6 @@ type QueryStateAndAuthChainResponse struct { IsRejected bool `json:"is_rejected"` } -// QueryRoomVersionCapabilitiesRequest asks for the default room version -type QueryRoomVersionCapabilitiesRequest struct{} - -// QueryRoomVersionCapabilitiesResponse is a response to QueryRoomVersionCapabilitiesRequest -type QueryRoomVersionCapabilitiesResponse struct { - DefaultRoomVersion gomatrixserverlib.RoomVersion `json:"default"` - AvailableRoomVersions map[gomatrixserverlib.RoomVersion]string `json:"available"` -} - // QueryRoomVersionForRoomRequest asks for the room version for a given room. type QueryRoomVersionForRoomRequest struct { RoomID string `json:"room_id"` diff --git a/roomserver/internal/query/query.go b/roomserver/internal/query/query.go index c5b74422f8..ce17580a17 100644 --- a/roomserver/internal/query/query.go +++ b/roomserver/internal/query/query.go @@ -35,7 +35,6 @@ import ( "github.com/matrix-org/dendrite/roomserver/state" "github.com/matrix-org/dendrite/roomserver/storage" "github.com/matrix-org/dendrite/roomserver/types" - "github.com/matrix-org/dendrite/roomserver/version" ) type Queryer struct { @@ -694,25 +693,7 @@ func GetAuthChain( return authEvents, nil } -// QueryRoomVersionCapabilities implements api.RoomserverInternalAPI -func (r *Queryer) QueryRoomVersionCapabilities( - ctx context.Context, - request *api.QueryRoomVersionCapabilitiesRequest, - response *api.QueryRoomVersionCapabilitiesResponse, -) error { - response.DefaultRoomVersion = version.DefaultRoomVersion() - response.AvailableRoomVersions = make(map[gomatrixserverlib.RoomVersion]string) - for v, desc := range version.SupportedRoomVersions() { - if desc.Stable { - response.AvailableRoomVersions[v] = "stable" - } else { - response.AvailableRoomVersions[v] = "unstable" - } - } - return nil -} - -// QueryRoomVersionCapabilities implements api.RoomserverInternalAPI +// QueryRoomVersionForRoom implements api.RoomserverInternalAPI func (r *Queryer) QueryRoomVersionForRoom( ctx context.Context, request *api.QueryRoomVersionForRoomRequest, From 985298cfc46cb6b33ab44d32a63a933d03e16429 Mon Sep 17 00:00:00 2001 From: Boris Rybalkin Date: Tue, 4 Apr 2023 08:42:46 +0100 Subject: [PATCH 31/42] app service unix socket support (#3022) This is the last part of unix socket support to talk to app servers, go based app services already support unix sockets: https://github.com/mautrix/go/commit/5a68173fe39345b8473e04bfa67cae5a13f6ca7f ``` appservice: # The address that the homeserver can use to connect to this appservice. address: unix:///var/snap/matrix/current/whatsapp.socket # The hostname and port where this appservice should listen. hostname: /var/snap/matrix/current/whatsapp.socket port: 0 ``` ### Pull Request Checklist * [x] I have added Go unit tests or [Complement integration tests](https://github.com/matrix-org/complement) for this PR _or_ I have justified why this PR doesn't need tests * [x] Pull request includes a [sign off below using a legally identifiable name](https://matrix-org.github.io/dendrite/development/contributing#sign-off) _or_ I have already signed off privately Signed-off-by: `Boris Rybalkin ` --- .gitignore | 1 + appservice/appservice.go | 17 +--- appservice/appservice_test.go | 151 ++++++++++++++++++++++++----- appservice/consumers/roomserver.go | 9 +- appservice/query/query.go | 21 ++-- setup/config/config_appservice.go | 51 +++++++++- 6 files changed, 189 insertions(+), 61 deletions(-) diff --git a/.gitignore b/.gitignore index fe5e827974..dcfbf80078 100644 --- a/.gitignore +++ b/.gitignore @@ -74,3 +74,4 @@ complement/ docs/_site media_store/ +build \ No newline at end of file diff --git a/appservice/appservice.go b/appservice/appservice.go index d13d9eb102..1f6037ee2e 100644 --- a/appservice/appservice.go +++ b/appservice/appservice.go @@ -16,10 +16,7 @@ package appservice import ( "context" - "crypto/tls" - "net/http" "sync" - "time" "github.com/matrix-org/dendrite/setup/jetstream" "github.com/matrix-org/dendrite/setup/process" @@ -44,20 +41,10 @@ func NewInternalAPI( userAPI userapi.AppserviceUserAPI, rsAPI roomserverAPI.RoomserverInternalAPI, ) appserviceAPI.AppServiceInternalAPI { - client := &http.Client{ - Timeout: time.Second * 30, - Transport: &http.Transport{ - DisableKeepAlives: true, - TLSClientConfig: &tls.Config{ - InsecureSkipVerify: cfg.AppServiceAPI.DisableTLSValidation, - }, - Proxy: http.ProxyFromEnvironment, - }, - } + // Create appserivce query API with an HTTP client that will be used for all // outbound and inbound requests (inbound only for the internal API) appserviceQueryAPI := &query.AppServiceQueryAPI{ - HTTPClient: client, Cfg: &cfg.AppServiceAPI, ProtocolCache: map[string]appserviceAPI.ASProtocolResponse{}, CacheMu: sync.Mutex{}, @@ -84,7 +71,7 @@ func NewInternalAPI( js, _ := natsInstance.Prepare(processContext, &cfg.Global.JetStream) consumer := consumers.NewOutputRoomEventConsumer( processContext, &cfg.AppServiceAPI, - client, js, rsAPI, + js, rsAPI, ) if err := consumer.Start(); err != nil { logrus.WithError(err).Panicf("failed to start appservice roomserver consumer") diff --git a/appservice/appservice_test.go b/appservice/appservice_test.go index 752901a9c5..5189bdf946 100644 --- a/appservice/appservice_test.go +++ b/appservice/appservice_test.go @@ -3,14 +3,19 @@ package appservice_test import ( "context" "encoding/json" + "fmt" + "net" "net/http" "net/http/httptest" + "path" "reflect" "regexp" "strings" "testing" "time" + "github.com/stretchr/testify/assert" + "github.com/matrix-org/dendrite/appservice" "github.com/matrix-org/dendrite/appservice/api" "github.com/matrix-org/dendrite/internal/caching" @@ -114,20 +119,20 @@ func TestAppserviceInternalAPI(t *testing.T) { defer close() // Create a dummy application service - cfg.AppServiceAPI.Derived.ApplicationServices = []config.ApplicationService{ - { - ID: "someID", - URL: srv.URL, - ASToken: "", - HSToken: "", - SenderLocalpart: "senderLocalPart", - NamespaceMap: map[string][]config.ApplicationServiceNamespace{ - "users": {{RegexpObject: regexp.MustCompile("as-.*")}}, - "aliases": {{RegexpObject: regexp.MustCompile("asroom-.*")}}, - }, - Protocols: []string{existingProtocol}, + as := &config.ApplicationService{ + ID: "someID", + URL: srv.URL, + ASToken: "", + HSToken: "", + SenderLocalpart: "senderLocalPart", + NamespaceMap: map[string][]config.ApplicationServiceNamespace{ + "users": {{RegexpObject: regexp.MustCompile("as-.*")}}, + "aliases": {{RegexpObject: regexp.MustCompile("asroom-.*")}}, }, + Protocols: []string{existingProtocol}, } + as.CreateHTTPClient(cfg.AppServiceAPI.DisableTLSValidation) + cfg.AppServiceAPI.Derived.ApplicationServices = []config.ApplicationService{*as} t.Cleanup(func() { ctx.ShutdownDendrite() @@ -145,6 +150,103 @@ func TestAppserviceInternalAPI(t *testing.T) { }) } +func TestAppserviceInternalAPI_UnixSocket_Simple(t *testing.T) { + + // Set expected results + existingProtocol := "irc" + wantLocationResponse := []api.ASLocationResponse{{Protocol: existingProtocol, Fields: []byte("{}")}} + wantUserResponse := []api.ASUserResponse{{Protocol: existingProtocol, Fields: []byte("{}")}} + wantProtocolResponse := api.ASProtocolResponse{Instances: []api.ProtocolInstance{{Fields: []byte("{}")}}} + + // create a dummy AS url, handling some cases + srv := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case strings.Contains(r.URL.Path, "location"): + // Check if we've got an existing protocol, if so, return a proper response. + if r.URL.Path[len(r.URL.Path)-len(existingProtocol):] == existingProtocol { + if err := json.NewEncoder(w).Encode(wantLocationResponse); err != nil { + t.Fatalf("failed to encode response: %s", err) + } + return + } + if err := json.NewEncoder(w).Encode([]api.ASLocationResponse{}); err != nil { + t.Fatalf("failed to encode response: %s", err) + } + return + case strings.Contains(r.URL.Path, "user"): + if r.URL.Path[len(r.URL.Path)-len(existingProtocol):] == existingProtocol { + if err := json.NewEncoder(w).Encode(wantUserResponse); err != nil { + t.Fatalf("failed to encode response: %s", err) + } + return + } + if err := json.NewEncoder(w).Encode([]api.UserResponse{}); err != nil { + t.Fatalf("failed to encode response: %s", err) + } + return + case strings.Contains(r.URL.Path, "protocol"): + if r.URL.Path[len(r.URL.Path)-len(existingProtocol):] == existingProtocol { + if err := json.NewEncoder(w).Encode(wantProtocolResponse); err != nil { + t.Fatalf("failed to encode response: %s", err) + } + return + } + if err := json.NewEncoder(w).Encode(nil); err != nil { + t.Fatalf("failed to encode response: %s", err) + } + return + default: + t.Logf("hit location: %s", r.URL.Path) + } + })) + + tmpDir := t.TempDir() + socket := path.Join(tmpDir, "socket") + l, err := net.Listen("unix", socket) + assert.NoError(t, err) + _ = srv.Listener.Close() + srv.Listener = l + srv.Start() + defer srv.Close() + + cfg, ctx, tearDown := testrig.CreateConfig(t, test.DBTypeSQLite) + defer tearDown() + + // Create a dummy application service + as := &config.ApplicationService{ + ID: "someID", + URL: fmt.Sprintf("unix://%s", socket), + ASToken: "", + HSToken: "", + SenderLocalpart: "senderLocalPart", + NamespaceMap: map[string][]config.ApplicationServiceNamespace{ + "users": {{RegexpObject: regexp.MustCompile("as-.*")}}, + "aliases": {{RegexpObject: regexp.MustCompile("asroom-.*")}}, + }, + Protocols: []string{existingProtocol}, + } + as.CreateHTTPClient(cfg.AppServiceAPI.DisableTLSValidation) + cfg.AppServiceAPI.Derived.ApplicationServices = []config.ApplicationService{*as} + + t.Cleanup(func() { + ctx.ShutdownDendrite() + ctx.WaitForShutdown() + }) + caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) + // Create required internal APIs + natsInstance := jetstream.NATSInstance{} + cm := sqlutil.NewConnectionManager(ctx, cfg.Global.DatabaseOptions) + rsAPI := roomserver.NewInternalAPI(ctx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) + usrAPI := userapi.NewInternalAPI(ctx, cfg, cm, &natsInstance, rsAPI, nil) + asAPI := appservice.NewInternalAPI(ctx, cfg, &natsInstance, usrAPI, rsAPI) + + t.Run("UserIDExists", func(t *testing.T) { + testUserIDExists(t, asAPI, "@as-testing:test", true) + testUserIDExists(t, asAPI, "@as1-testing:test", false) + }) + +} + func testUserIDExists(t *testing.T, asAPI api.AppServiceInternalAPI, userID string, wantExists bool) { ctx := context.Background() userResp := &api.UserIDExistsResponse{} @@ -254,20 +356,21 @@ func TestRoomserverConsumerOneInvite(t *testing.T) { })) defer srv.Close() - // Create a dummy application service - cfg.AppServiceAPI.Derived.ApplicationServices = []config.ApplicationService{ - { - ID: "someID", - URL: srv.URL, - ASToken: "", - HSToken: "", - SenderLocalpart: "senderLocalPart", - NamespaceMap: map[string][]config.ApplicationServiceNamespace{ - "users": {{RegexpObject: regexp.MustCompile(bob.ID)}}, - "aliases": {{RegexpObject: regexp.MustCompile(room.ID)}}, - }, + as := &config.ApplicationService{ + ID: "someID", + URL: srv.URL, + ASToken: "", + HSToken: "", + SenderLocalpart: "senderLocalPart", + NamespaceMap: map[string][]config.ApplicationServiceNamespace{ + "users": {{RegexpObject: regexp.MustCompile(bob.ID)}}, + "aliases": {{RegexpObject: regexp.MustCompile(room.ID)}}, }, } + as.CreateHTTPClient(cfg.AppServiceAPI.DisableTLSValidation) + + // Create a dummy application service + cfg.AppServiceAPI.Derived.ApplicationServices = []config.ApplicationService{*as} caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) // Create required internal APIs diff --git a/appservice/consumers/roomserver.go b/appservice/consumers/roomserver.go index 16b3b82319..308b036793 100644 --- a/appservice/consumers/roomserver.go +++ b/appservice/consumers/roomserver.go @@ -40,7 +40,6 @@ import ( type OutputRoomEventConsumer struct { ctx context.Context cfg *config.AppServiceAPI - client *http.Client jetstream nats.JetStreamContext topic string rsAPI api.AppserviceRoomserverAPI @@ -56,14 +55,12 @@ type appserviceState struct { func NewOutputRoomEventConsumer( process *process.ProcessContext, cfg *config.AppServiceAPI, - client *http.Client, js nats.JetStreamContext, rsAPI api.AppserviceRoomserverAPI, ) *OutputRoomEventConsumer { return &OutputRoomEventConsumer{ ctx: process.Context(), cfg: cfg, - client: client, jetstream: js, topic: cfg.Matrix.JetStream.Prefixed(jetstream.OutputRoomEvent), rsAPI: rsAPI, @@ -189,13 +186,13 @@ func (s *OutputRoomEventConsumer) sendEvents( // Send the transaction to the appservice. // https://matrix.org/docs/spec/application_service/r0.1.2#put-matrix-app-v1-transactions-txnid - address := fmt.Sprintf("%s/transactions/%s?access_token=%s", state.URL, txnID, url.QueryEscape(state.HSToken)) + address := fmt.Sprintf("%s/transactions/%s?access_token=%s", state.RequestUrl(), txnID, url.QueryEscape(state.HSToken)) req, err := http.NewRequestWithContext(ctx, "PUT", address, bytes.NewBuffer(transaction)) if err != nil { return err } req.Header.Set("Content-Type", "application/json") - resp, err := s.client.Do(req) + resp, err := state.HTTPClient.Do(req) if err != nil { return state.backoffAndPause(err) } @@ -206,7 +203,7 @@ func (s *OutputRoomEventConsumer) sendEvents( case http.StatusOK: state.backoff = 0 default: - return state.backoffAndPause(fmt.Errorf("received HTTP status code %d from appservice", resp.StatusCode)) + return state.backoffAndPause(fmt.Errorf("received HTTP status code %d from appservice url %s", resp.StatusCode, address)) } return nil } diff --git a/appservice/query/query.go b/appservice/query/query.go index 0466f81d0c..ca8d7b3a32 100644 --- a/appservice/query/query.go +++ b/appservice/query/query.go @@ -37,7 +37,6 @@ const userIDExistsPath = "/users/" // AppServiceQueryAPI is an implementation of api.AppServiceQueryAPI type AppServiceQueryAPI struct { - HTTPClient *http.Client Cfg *config.AppServiceAPI ProtocolCache map[string]api.ASProtocolResponse CacheMu sync.Mutex @@ -57,7 +56,7 @@ func (a *AppServiceQueryAPI) RoomAliasExists( for _, appservice := range a.Cfg.Derived.ApplicationServices { if appservice.URL != "" && appservice.IsInterestedInRoomAlias(request.Alias) { // The full path to the rooms API, includes hs token - URL, err := url.Parse(appservice.URL + roomAliasExistsPath) + URL, err := url.Parse(appservice.RequestUrl() + roomAliasExistsPath) if err != nil { return err } @@ -73,7 +72,7 @@ func (a *AppServiceQueryAPI) RoomAliasExists( } req = req.WithContext(ctx) - resp, err := a.HTTPClient.Do(req) + resp, err := appservice.HTTPClient.Do(req) if resp != nil { defer func() { err = resp.Body.Close() @@ -124,7 +123,7 @@ func (a *AppServiceQueryAPI) UserIDExists( for _, appservice := range a.Cfg.Derived.ApplicationServices { if appservice.URL != "" && appservice.IsInterestedInUserID(request.UserID) { // The full path to the rooms API, includes hs token - URL, err := url.Parse(appservice.URL + userIDExistsPath) + URL, err := url.Parse(appservice.RequestUrl() + userIDExistsPath) if err != nil { return err } @@ -137,7 +136,7 @@ func (a *AppServiceQueryAPI) UserIDExists( if err != nil { return err } - resp, err := a.HTTPClient.Do(req.WithContext(ctx)) + resp, err := appservice.HTTPClient.Do(req.WithContext(ctx)) if resp != nil { defer func() { err = resp.Body.Close() @@ -212,12 +211,12 @@ func (a *AppServiceQueryAPI) Locations( var asLocations []api.ASLocationResponse params.Set("access_token", as.HSToken) - url := as.URL + api.ASLocationPath + url := as.RequestUrl() + api.ASLocationPath if req.Protocol != "" { url += "/" + req.Protocol } - if err := requestDo[[]api.ASLocationResponse](a.HTTPClient, url+"?"+params.Encode(), &asLocations); err != nil { + if err := requestDo[[]api.ASLocationResponse](as.HTTPClient, url+"?"+params.Encode(), &asLocations); err != nil { log.WithError(err).Error("unable to get 'locations' from application service") continue } @@ -247,12 +246,12 @@ func (a *AppServiceQueryAPI) User( var asUsers []api.ASUserResponse params.Set("access_token", as.HSToken) - url := as.URL + api.ASUserPath + url := as.RequestUrl() + api.ASUserPath if req.Protocol != "" { url += "/" + req.Protocol } - if err := requestDo[[]api.ASUserResponse](a.HTTPClient, url+"?"+params.Encode(), &asUsers); err != nil { + if err := requestDo[[]api.ASUserResponse](as.HTTPClient, url+"?"+params.Encode(), &asUsers); err != nil { log.WithError(err).Error("unable to get 'user' from application service") continue } @@ -290,7 +289,7 @@ func (a *AppServiceQueryAPI) Protocols( response := api.ASProtocolResponse{} for _, as := range a.Cfg.Derived.ApplicationServices { var proto api.ASProtocolResponse - if err := requestDo[api.ASProtocolResponse](a.HTTPClient, as.URL+api.ASProtocolPath+req.Protocol, &proto); err != nil { + if err := requestDo[api.ASProtocolResponse](as.HTTPClient, as.RequestUrl()+api.ASProtocolPath+req.Protocol, &proto); err != nil { log.WithError(err).Error("unable to get 'protocol' from application service") continue } @@ -320,7 +319,7 @@ func (a *AppServiceQueryAPI) Protocols( for _, as := range a.Cfg.Derived.ApplicationServices { for _, p := range as.Protocols { var proto api.ASProtocolResponse - if err := requestDo[api.ASProtocolResponse](a.HTTPClient, as.URL+api.ASProtocolPath+p, &proto); err != nil { + if err := requestDo[api.ASProtocolResponse](as.HTTPClient, as.RequestUrl()+api.ASProtocolPath+p, &proto); err != nil { log.WithError(err).Error("unable to get 'protocol' from application service") continue } diff --git a/setup/config/config_appservice.go b/setup/config/config_appservice.go index 37e20a978b..ef10649d25 100644 --- a/setup/config/config_appservice.go +++ b/setup/config/config_appservice.go @@ -15,16 +15,23 @@ package config import ( + "context" + "crypto/tls" "fmt" + "net" + "net/http" "os" "path/filepath" "regexp" "strings" + "time" log "github.com/sirupsen/logrus" "gopkg.in/yaml.v2" ) +const UnixSocketPrefix = "unix://" + type AppServiceAPI struct { Matrix *Global `yaml:"-"` Derived *Derived `yaml:"-"` // TODO: Nuke Derived from orbit @@ -80,7 +87,41 @@ type ApplicationService struct { // Whether rate limiting is applied to each application service user RateLimited bool `yaml:"rate_limited"` // Any custom protocols that this application service provides (e.g. IRC) - Protocols []string `yaml:"protocols"` + Protocols []string `yaml:"protocols"` + HTTPClient *http.Client + isUnixSocket bool + unixSocket string +} + +func (a *ApplicationService) CreateHTTPClient(insecureSkipVerify bool) { + client := &http.Client{ + Timeout: time.Second * 30, + Transport: &http.Transport{ + DisableKeepAlives: true, + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: insecureSkipVerify, + }, + Proxy: http.ProxyFromEnvironment, + }, + } + if strings.HasPrefix(a.URL, UnixSocketPrefix) { + a.isUnixSocket = true + a.unixSocket = "http://unix" + client.Transport = &http.Transport{ + DialContext: func(_ context.Context, _, _ string) (net.Conn, error) { + return net.Dial("unix", strings.TrimPrefix(a.URL, UnixSocketPrefix)) + }, + } + } + a.HTTPClient = client +} + +func (a *ApplicationService) RequestUrl() string { + if a.isUnixSocket { + return a.unixSocket + } else { + return a.URL + } } // IsInterestedInRoomID returns a bool on whether an application service's @@ -152,7 +193,7 @@ func (a *ApplicationService) IsInterestedInRoomAlias( func loadAppServices(config *AppServiceAPI, derived *Derived) error { for _, configPath := range config.ConfigFiles { // Create a new application service with default options - appservice := ApplicationService{ + appservice := &ApplicationService{ RateLimited: true, } @@ -169,13 +210,13 @@ func loadAppServices(config *AppServiceAPI, derived *Derived) error { } // Load the config data into our struct - if err = yaml.Unmarshal(configData, &appservice); err != nil { + if err = yaml.Unmarshal(configData, appservice); err != nil { return err } - + appservice.CreateHTTPClient(config.DisableTLSValidation) // Append the parsed application service to the global config derived.ApplicationServices = append( - derived.ApplicationServices, appservice, + derived.ApplicationServices, *appservice, ) } From 3691423626c94418388c48c1a91ac69dcd8b196b Mon Sep 17 00:00:00 2001 From: kegsay Date: Tue, 4 Apr 2023 18:16:53 +0100 Subject: [PATCH 32/42] Move GMSL client types to Dendrite (#3045) GMSL is intended for Federation only. Sister PR to https://github.com/matrix-org/gomatrixserverlib/pull/357 --- appservice/appservice_test.go | 3 +- appservice/consumers/roomserver.go | 11 +- clientapi/routing/state.go | 11 +- roomserver/api/query.go | 3 +- roomserver/internal/query/query.go | 7 +- setup/mscs/msc2836/msc2836.go | 9 +- setup/mscs/msc2836/msc2836_test.go | 3 +- syncapi/internal/keychange.go | 3 +- syncapi/internal/keychange_test.go | 11 +- syncapi/routing/context.go | 33 ++-- syncapi/routing/context_test.go | 12 +- syncapi/routing/filter.go | 5 +- syncapi/routing/getevent.go | 4 +- syncapi/routing/memberships.go | 6 +- syncapi/routing/messages.go | 25 +-- syncapi/routing/relations.go | 13 +- syncapi/routing/search.go | 59 +++---- syncapi/routing/search_test.go | 13 +- syncapi/storage/interface.go | 25 +-- .../storage/postgres/account_data_table.go | 4 +- .../postgres/current_room_state_table.go | 3 +- syncapi/storage/postgres/filter_table.go | 5 +- syncapi/storage/postgres/filtering.go | 6 +- .../postgres/output_room_events_table.go | 13 +- syncapi/storage/postgres/presence_table.go | 3 +- syncapi/storage/shared/storage_consumer.go | 9 +- syncapi/storage/shared/storage_sync.go | 23 +-- syncapi/storage/shared/storage_sync_test.go | 18 +-- syncapi/storage/sqlite3/account_data_table.go | 4 +- .../sqlite3/current_room_state_table.go | 3 +- syncapi/storage/sqlite3/filter_table.go | 5 +- .../sqlite3/output_room_events_table.go | 15 +- syncapi/storage/sqlite3/presence_table.go | 3 +- syncapi/storage/storage_test.go | 7 +- .../storage/tables/current_room_state_test.go | 3 +- syncapi/storage/tables/interface.go | 23 +-- .../storage/tables/output_room_events_test.go | 3 +- syncapi/storage/tables/presence_table_test.go | 3 +- syncapi/streams/stream_accountdata.go | 5 +- syncapi/streams/stream_invite.go | 3 +- syncapi/streams/stream_pdu.go | 31 ++-- syncapi/streams/stream_presence.go | 7 +- syncapi/streams/stream_receipt.go | 3 +- syncapi/streams/stream_typing.go | 3 +- syncapi/sync/request.go | 3 +- syncapi/sync/requestpool_test.go | 3 +- syncapi/syncapi_test.go | 5 +- syncapi/synctypes/clientevent.go | 88 ++++++++++ syncapi/synctypes/clientevent_test.go | 105 ++++++++++++ syncapi/synctypes/filter.go | 152 ++++++++++++++++++ syncapi/synctypes/filter_test.go | 60 +++++++ syncapi/types/provider.go | 3 +- syncapi/types/types.go | 11 +- syncapi/types/types_test.go | 5 +- userapi/api/api.go | 13 +- userapi/consumers/roomserver.go | 7 +- userapi/storage/storage_test.go | 7 +- userapi/util/notify_test.go | 3 +- 58 files changed, 692 insertions(+), 234 deletions(-) create mode 100644 syncapi/synctypes/clientevent.go create mode 100644 syncapi/synctypes/clientevent_test.go create mode 100644 syncapi/synctypes/filter.go create mode 100644 syncapi/synctypes/filter_test.go diff --git a/appservice/appservice_test.go b/appservice/appservice_test.go index 5189bdf946..282c631285 100644 --- a/appservice/appservice_test.go +++ b/appservice/appservice_test.go @@ -18,6 +18,7 @@ import ( "github.com/matrix-org/dendrite/appservice" "github.com/matrix-org/dendrite/appservice/api" + "github.com/matrix-org/dendrite/appservice/consumers" "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver" @@ -338,7 +339,7 @@ func TestRoomserverConsumerOneInvite(t *testing.T) { evChan := make(chan struct{}) // create a dummy AS url, handling the events srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - var txn gomatrixserverlib.ApplicationServiceTransaction + var txn consumers.ApplicationServiceTransaction err := json.NewDecoder(r.Body).Decode(&txn) if err != nil { t.Fatal(err) diff --git a/appservice/consumers/roomserver.go b/appservice/consumers/roomserver.go index 308b036793..586ca33a87 100644 --- a/appservice/consumers/roomserver.go +++ b/appservice/consumers/roomserver.go @@ -32,10 +32,17 @@ import ( "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/jetstream" "github.com/matrix-org/dendrite/setup/process" + "github.com/matrix-org/dendrite/syncapi/synctypes" log "github.com/sirupsen/logrus" ) +// ApplicationServiceTransaction is the transaction that is sent off to an +// application service. +type ApplicationServiceTransaction struct { + Events []synctypes.ClientEvent `json:"events"` +} + // OutputRoomEventConsumer consumes events that originated in the room server. type OutputRoomEventConsumer struct { ctx context.Context @@ -171,8 +178,8 @@ func (s *OutputRoomEventConsumer) sendEvents( ) error { // Create the transaction body. transaction, err := json.Marshal( - gomatrixserverlib.ApplicationServiceTransaction{ - Events: gomatrixserverlib.HeaderedToClientEvents(events, gomatrixserverlib.FormatAll), + ApplicationServiceTransaction{ + Events: synctypes.HeaderedToClientEvents(events, synctypes.FormatAll), }, ) if err != nil { diff --git a/clientapi/routing/state.go b/clientapi/routing/state.go index 12984c39a7..59ed9a5f6e 100644 --- a/clientapi/routing/state.go +++ b/clientapi/routing/state.go @@ -22,6 +22,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/syncapi/synctypes" userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" @@ -29,7 +30,7 @@ import ( ) type stateEventInStateResp struct { - gomatrixserverlib.ClientEvent + synctypes.ClientEvent PrevContent json.RawMessage `json:"prev_content,omitempty"` ReplacesState string `json:"replaces_state,omitempty"` } @@ -122,7 +123,7 @@ func OnIncomingStateRequest(ctx context.Context, device *userapi.Device, rsAPI a "state_at_event": !wantLatestState, }).Info("Fetching all state") - stateEvents := []gomatrixserverlib.ClientEvent{} + stateEvents := []synctypes.ClientEvent{} if wantLatestState { // If we are happy to use the latest state, either because the user is // still in the room, or because the room is world-readable, then just @@ -131,7 +132,7 @@ func OnIncomingStateRequest(ctx context.Context, device *userapi.Device, rsAPI a for _, ev := range stateRes.StateEvents { stateEvents = append( stateEvents, - gomatrixserverlib.HeaderedToClientEvent(ev, gomatrixserverlib.FormatAll), + synctypes.HeaderedToClientEvent(ev, synctypes.FormatAll), ) } } else { @@ -150,7 +151,7 @@ func OnIncomingStateRequest(ctx context.Context, device *userapi.Device, rsAPI a for _, ev := range stateAfterRes.StateEvents { stateEvents = append( stateEvents, - gomatrixserverlib.HeaderedToClientEvent(ev, gomatrixserverlib.FormatAll), + synctypes.HeaderedToClientEvent(ev, synctypes.FormatAll), ) } } @@ -309,7 +310,7 @@ func OnIncomingStateTypeRequest( } stateEvent := stateEventInStateResp{ - ClientEvent: gomatrixserverlib.HeaderedToClientEvent(event, gomatrixserverlib.FormatAll), + ClientEvent: synctypes.HeaderedToClientEvent(event, synctypes.FormatAll), } var res interface{} diff --git a/roomserver/api/query.go b/roomserver/api/query.go index d60ec4f60e..612c331567 100644 --- a/roomserver/api/query.go +++ b/roomserver/api/query.go @@ -24,6 +24,7 @@ import ( "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" + "github.com/matrix-org/dendrite/syncapi/synctypes" ) // QueryLatestEventsAndStateRequest is a request to QueryLatestEventsAndState @@ -146,7 +147,7 @@ type QueryMembershipsForRoomRequest struct { // QueryMembershipsForRoomResponse is a response to QueryMembershipsForRoom type QueryMembershipsForRoomResponse struct { // The "m.room.member" events (of "join" membership) in the client format - JoinEvents []gomatrixserverlib.ClientEvent `json:"join_events"` + JoinEvents []synctypes.ClientEvent `json:"join_events"` // True if the user has been in room before and has either stayed in it or // left it. HasBeenInRoom bool `json:"has_been_in_room"` diff --git a/roomserver/internal/query/query.go b/roomserver/internal/query/query.go index ce17580a17..cac8d995f8 100644 --- a/roomserver/internal/query/query.go +++ b/roomserver/internal/query/query.go @@ -26,6 +26,7 @@ import ( "github.com/sirupsen/logrus" "github.com/matrix-org/dendrite/roomserver/storage/tables" + "github.com/matrix-org/dendrite/syncapi/synctypes" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/internal/caching" @@ -345,7 +346,7 @@ func (r *Queryer) QueryMembershipsForRoom( return fmt.Errorf("r.DB.Events: %w", err) } for _, event := range events { - clientEvent := gomatrixserverlib.ToClientEvent(event.Event, gomatrixserverlib.FormatAll) + clientEvent := synctypes.ToClientEvent(event.Event, synctypes.FormatAll) response.JoinEvents = append(response.JoinEvents, clientEvent) } return nil @@ -365,7 +366,7 @@ func (r *Queryer) QueryMembershipsForRoom( } response.HasBeenInRoom = true - response.JoinEvents = []gomatrixserverlib.ClientEvent{} + response.JoinEvents = []synctypes.ClientEvent{} var events []types.Event var stateEntries []types.StateEntry @@ -394,7 +395,7 @@ func (r *Queryer) QueryMembershipsForRoom( } for _, event := range events { - clientEvent := gomatrixserverlib.ToClientEvent(event.Event, gomatrixserverlib.FormatAll) + clientEvent := synctypes.ToClientEvent(event.Event, synctypes.FormatAll) response.JoinEvents = append(response.JoinEvents, clientEvent) } diff --git a/setup/mscs/msc2836/msc2836.go b/setup/mscs/msc2836/msc2836.go index 92248bc101..2612f9117e 100644 --- a/setup/mscs/msc2836/msc2836.go +++ b/setup/mscs/msc2836/msc2836.go @@ -34,6 +34,7 @@ import ( "github.com/matrix-org/dendrite/internal/sqlutil" roomserver "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/dendrite/syncapi/synctypes" userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" @@ -78,9 +79,9 @@ func (r *EventRelationshipRequest) Defaults() { } type EventRelationshipResponse struct { - Events []gomatrixserverlib.ClientEvent `json:"events"` - NextBatch string `json:"next_batch"` - Limited bool `json:"limited"` + Events []synctypes.ClientEvent `json:"events"` + NextBatch string `json:"next_batch"` + Limited bool `json:"limited"` } type MSC2836EventRelationshipsResponse struct { @@ -91,7 +92,7 @@ type MSC2836EventRelationshipsResponse struct { func toClientResponse(res *MSC2836EventRelationshipsResponse) *EventRelationshipResponse { out := &EventRelationshipResponse{ - Events: gomatrixserverlib.ToClientEvents(res.ParsedEvents, gomatrixserverlib.FormatAll), + Events: synctypes.ToClientEvents(res.ParsedEvents, synctypes.FormatAll), Limited: res.Limited, NextBatch: res.NextBatch, } diff --git a/setup/mscs/msc2836/msc2836_test.go b/setup/mscs/msc2836/msc2836_test.go index bfcabef640..3c4431489f 100644 --- a/setup/mscs/msc2836/msc2836_test.go +++ b/setup/mscs/msc2836/msc2836_test.go @@ -16,6 +16,7 @@ import ( "github.com/gorilla/mux" "github.com/matrix-org/dendrite/setup/process" + "github.com/matrix-org/dendrite/syncapi/synctypes" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/dendrite/internal/hooks" @@ -462,7 +463,7 @@ func assertContains(t *testing.T, result *msc2836.EventRelationshipResponse, wan } } -func assertUnsignedChildren(t *testing.T, ev gomatrixserverlib.ClientEvent, relType string, wantCount int, childrenEventIDs []string) { +func assertUnsignedChildren(t *testing.T, ev synctypes.ClientEvent, relType string, wantCount int, childrenEventIDs []string) { t.Helper() unsigned := struct { Children map[string]int `json:"children"` diff --git a/syncapi/internal/keychange.go b/syncapi/internal/keychange.go index e7f677c85c..17d63708a6 100644 --- a/syncapi/internal/keychange.go +++ b/syncapi/internal/keychange.go @@ -26,6 +26,7 @@ import ( roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/syncapi/storage" + "github.com/matrix-org/dendrite/syncapi/synctypes" "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/dendrite/userapi/api" ) @@ -278,7 +279,7 @@ func leftRooms(res *types.Response) []string { return roomIDs } -func membershipEventPresent(events []gomatrixserverlib.ClientEvent, userID string) bool { +func membershipEventPresent(events []synctypes.ClientEvent, userID string) bool { for _, ev := range events { // it's enough to know that we have our member event here, don't need to check membership content // as it's implied by being in the respective section of the sync response. diff --git a/syncapi/internal/keychange_test.go b/syncapi/internal/keychange_test.go index 4bb8516684..f775276fee 100644 --- a/syncapi/internal/keychange_test.go +++ b/syncapi/internal/keychange_test.go @@ -10,6 +10,7 @@ import ( "github.com/matrix-org/util" "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/syncapi/synctypes" "github.com/matrix-org/dendrite/syncapi/types" userapi "github.com/matrix-org/dendrite/userapi/api" ) @@ -159,7 +160,7 @@ func assertCatchup(t *testing.T, hasNew bool, syncResponse *types.Response, want func joinResponseWithRooms(syncResponse *types.Response, userID string, roomIDs []string) *types.Response { for _, roomID := range roomIDs { - roomEvents := []gomatrixserverlib.ClientEvent{ + roomEvents := []synctypes.ClientEvent{ { Type: "m.room.member", StateKey: &userID, @@ -182,7 +183,7 @@ func joinResponseWithRooms(syncResponse *types.Response, userID string, roomIDs func leaveResponseWithRooms(syncResponse *types.Response, userID string, roomIDs []string) *types.Response { for _, roomID := range roomIDs { - roomEvents := []gomatrixserverlib.ClientEvent{ + roomEvents := []synctypes.ClientEvent{ { Type: "m.room.member", StateKey: &userID, @@ -299,7 +300,7 @@ func TestKeyChangeCatchupNoNewJoinsButMessages(t *testing.T) { roomID := "!TestKeyChangeCatchupNoNewJoinsButMessages:bar" syncResponse := types.NewResponse() empty := "" - roomStateEvents := []gomatrixserverlib.ClientEvent{ + roomStateEvents := []synctypes.ClientEvent{ { Type: "m.room.name", StateKey: &empty, @@ -309,7 +310,7 @@ func TestKeyChangeCatchupNoNewJoinsButMessages(t *testing.T) { Content: []byte(`{"name":"The Room Name"}`), }, } - roomTimelineEvents := []gomatrixserverlib.ClientEvent{ + roomTimelineEvents := []synctypes.ClientEvent{ { Type: "m.room.message", EventID: "$something1:here", @@ -402,7 +403,7 @@ func TestKeyChangeCatchupChangeAndLeftSameRoom(t *testing.T) { newShareUser2 := "@bobby:localhost" roomID := "!join:bar" syncResponse := types.NewResponse() - roomEvents := []gomatrixserverlib.ClientEvent{ + roomEvents := []synctypes.ClientEvent{ { Type: "m.room.member", StateKey: &syncingUser, diff --git a/syncapi/routing/context.go b/syncapi/routing/context.go index 76f0036710..dd42c7ac41 100644 --- a/syncapi/routing/context.go +++ b/syncapi/routing/context.go @@ -29,6 +29,7 @@ import ( roomserver "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/syncapi/internal" "github.com/matrix-org/dendrite/syncapi/storage" + "github.com/matrix-org/dendrite/syncapi/synctypes" "github.com/matrix-org/dendrite/syncapi/types" userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" @@ -37,12 +38,12 @@ import ( ) type ContextRespsonse struct { - End string `json:"end"` - Event *gomatrixserverlib.ClientEvent `json:"event,omitempty"` - EventsAfter []gomatrixserverlib.ClientEvent `json:"events_after,omitempty"` - EventsBefore []gomatrixserverlib.ClientEvent `json:"events_before,omitempty"` - Start string `json:"start"` - State []gomatrixserverlib.ClientEvent `json:"state,omitempty"` + End string `json:"end"` + Event *synctypes.ClientEvent `json:"event,omitempty"` + EventsAfter []synctypes.ClientEvent `json:"events_after,omitempty"` + EventsBefore []synctypes.ClientEvent `json:"events_before,omitempty"` + Start string `json:"start"` + State []synctypes.ClientEvent `json:"state,omitempty"` } func Context( @@ -94,7 +95,7 @@ func Context( } } - stateFilter := gomatrixserverlib.StateFilter{ + stateFilter := synctypes.StateFilter{ NotSenders: filter.NotSenders, NotTypes: filter.NotTypes, Senders: filter.Senders, @@ -167,14 +168,14 @@ func Context( return jsonerror.InternalServerError() } - eventsBeforeClient := gomatrixserverlib.HeaderedToClientEvents(eventsBeforeFiltered, gomatrixserverlib.FormatAll) - eventsAfterClient := gomatrixserverlib.HeaderedToClientEvents(eventsAfterFiltered, gomatrixserverlib.FormatAll) + eventsBeforeClient := synctypes.HeaderedToClientEvents(eventsBeforeFiltered, synctypes.FormatAll) + eventsAfterClient := synctypes.HeaderedToClientEvents(eventsAfterFiltered, synctypes.FormatAll) newState := state if filter.LazyLoadMembers { allEvents := append(eventsBeforeFiltered, eventsAfterFiltered...) allEvents = append(allEvents, &requestedEvent) - evs := gomatrixserverlib.HeaderedToClientEvents(allEvents, gomatrixserverlib.FormatAll) + evs := synctypes.HeaderedToClientEvents(allEvents, synctypes.FormatAll) newState, err = applyLazyLoadMembers(ctx, device, snapshot, roomID, evs, lazyLoadCache) if err != nil { logrus.WithError(err).Error("unable to load membership events") @@ -182,12 +183,12 @@ func Context( } } - ev := gomatrixserverlib.HeaderedToClientEvent(&requestedEvent, gomatrixserverlib.FormatAll) + ev := synctypes.HeaderedToClientEvent(&requestedEvent, synctypes.FormatAll) response := ContextRespsonse{ Event: &ev, EventsAfter: eventsAfterClient, EventsBefore: eventsBeforeClient, - State: gomatrixserverlib.HeaderedToClientEvents(newState, gomatrixserverlib.FormatAll), + State: synctypes.HeaderedToClientEvents(newState, synctypes.FormatAll), } if len(response.State) > filter.Limit { @@ -261,7 +262,7 @@ func applyLazyLoadMembers( device *userapi.Device, snapshot storage.DatabaseTransaction, roomID string, - events []gomatrixserverlib.ClientEvent, + events []synctypes.ClientEvent, lazyLoadCache caching.LazyLoadCache, ) ([]*gomatrixserverlib.HeaderedEvent, error) { eventSenders := make(map[string]struct{}) @@ -280,7 +281,7 @@ func applyLazyLoadMembers( } // Query missing membership events - filter := gomatrixserverlib.DefaultStateFilter() + filter := synctypes.DefaultStateFilter() filter.Senders = &wantUsers filter.Types = &[]string{gomatrixserverlib.MRoomMember} memberships, err := snapshot.GetStateEventsForRoom(ctx, roomID, &filter) @@ -296,9 +297,9 @@ func applyLazyLoadMembers( return memberships, nil } -func parseRoomEventFilter(req *http.Request) (*gomatrixserverlib.RoomEventFilter, error) { +func parseRoomEventFilter(req *http.Request) (*synctypes.RoomEventFilter, error) { // Default room filter - filter := &gomatrixserverlib.RoomEventFilter{Limit: 10} + filter := &synctypes.RoomEventFilter{Limit: 10} l := req.URL.Query().Get("limit") f := req.URL.Query().Get("filter") diff --git a/syncapi/routing/context_test.go b/syncapi/routing/context_test.go index e79a5d5f19..5c6682bd13 100644 --- a/syncapi/routing/context_test.go +++ b/syncapi/routing/context_test.go @@ -5,7 +5,7 @@ import ( "reflect" "testing" - "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/dendrite/syncapi/synctypes" ) func Test_parseContextParams(t *testing.T) { @@ -19,28 +19,28 @@ func Test_parseContextParams(t *testing.T) { tests := []struct { name string req *http.Request - wantFilter *gomatrixserverlib.RoomEventFilter + wantFilter *synctypes.RoomEventFilter wantErr bool }{ { name: "no params set", req: noParamsReq, - wantFilter: &gomatrixserverlib.RoomEventFilter{Limit: 10}, + wantFilter: &synctypes.RoomEventFilter{Limit: 10}, }, { name: "limit 2 param set", req: limit2Req, - wantFilter: &gomatrixserverlib.RoomEventFilter{Limit: 2}, + wantFilter: &synctypes.RoomEventFilter{Limit: 2}, }, { name: "limit 10000 param set", req: limit10000Req, - wantFilter: &gomatrixserverlib.RoomEventFilter{Limit: 100}, + wantFilter: &synctypes.RoomEventFilter{Limit: 100}, }, { name: "filter lazy_load_members param set", req: lazyLoadReq, - wantFilter: &gomatrixserverlib.RoomEventFilter{Limit: 2, LazyLoadMembers: true}, + wantFilter: &synctypes.RoomEventFilter{Limit: 2, LazyLoadMembers: true}, }, { name: "invalid limit req", diff --git a/syncapi/routing/filter.go b/syncapi/routing/filter.go index f5acdbde3f..266ad4adcc 100644 --- a/syncapi/routing/filter.go +++ b/syncapi/routing/filter.go @@ -26,6 +26,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/syncapi/storage" "github.com/matrix-org/dendrite/syncapi/sync" + "github.com/matrix-org/dendrite/syncapi/synctypes" "github.com/matrix-org/dendrite/userapi/api" ) @@ -45,7 +46,7 @@ func GetFilter( return jsonerror.InternalServerError() } - filter := gomatrixserverlib.DefaultFilter() + filter := synctypes.DefaultFilter() if err := syncDB.GetFilter(req.Context(), &filter, localpart, filterID); err != nil { //TODO better error handling. This error message is *probably* right, // but if there are obscure db errors, this will also be returned, @@ -85,7 +86,7 @@ func PutFilter( return jsonerror.InternalServerError() } - var filter gomatrixserverlib.Filter + var filter synctypes.Filter defer req.Body.Close() // nolint:errcheck body, err := io.ReadAll(req.Body) diff --git a/syncapi/routing/getevent.go b/syncapi/routing/getevent.go index d2cdc1b5fd..84986d3b3f 100644 --- a/syncapi/routing/getevent.go +++ b/syncapi/routing/getevent.go @@ -17,7 +17,6 @@ package routing import ( "net/http" - "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" "github.com/sirupsen/logrus" @@ -26,6 +25,7 @@ import ( "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/syncapi/internal" "github.com/matrix-org/dendrite/syncapi/storage" + "github.com/matrix-org/dendrite/syncapi/synctypes" userapi "github.com/matrix-org/dendrite/userapi/api" ) @@ -97,6 +97,6 @@ func GetEvent( return util.JSONResponse{ Code: http.StatusOK, - JSON: gomatrixserverlib.HeaderedToClientEvent(events[0], gomatrixserverlib.FormatAll), + JSON: synctypes.HeaderedToClientEvent(events[0], synctypes.FormatAll), } } diff --git a/syncapi/routing/memberships.go b/syncapi/routing/memberships.go index 8efd77cef1..9ea660f596 100644 --- a/syncapi/routing/memberships.go +++ b/syncapi/routing/memberships.go @@ -22,14 +22,14 @@ import ( "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/syncapi/storage" + "github.com/matrix-org/dendrite/syncapi/synctypes" "github.com/matrix-org/dendrite/syncapi/types" userapi "github.com/matrix-org/dendrite/userapi/api" - "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" ) type getMembershipResponse struct { - Chunk []gomatrixserverlib.ClientEvent `json:"chunk"` + Chunk []synctypes.ClientEvent `json:"chunk"` } // https://matrix.org/docs/spec/client_server/r0.6.0#get-matrix-client-r0-rooms-roomid-joined-members @@ -134,6 +134,6 @@ func GetMemberships( } return util.JSONResponse{ Code: http.StatusOK, - JSON: getMembershipResponse{gomatrixserverlib.HeaderedToClientEvents(result, gomatrixserverlib.FormatAll)}, + JSON: getMembershipResponse{synctypes.HeaderedToClientEvents(result, synctypes.FormatAll)}, } } diff --git a/syncapi/routing/messages.go b/syncapi/routing/messages.go index 02d8fcc7ea..3c41662728 100644 --- a/syncapi/routing/messages.go +++ b/syncapi/routing/messages.go @@ -34,6 +34,7 @@ import ( "github.com/matrix-org/dendrite/syncapi/internal" "github.com/matrix-org/dendrite/syncapi/storage" "github.com/matrix-org/dendrite/syncapi/sync" + "github.com/matrix-org/dendrite/syncapi/synctypes" "github.com/matrix-org/dendrite/syncapi/types" userapi "github.com/matrix-org/dendrite/userapi/api" ) @@ -50,15 +51,15 @@ type messagesReq struct { device *userapi.Device wasToProvided bool backwardOrdering bool - filter *gomatrixserverlib.RoomEventFilter + filter *synctypes.RoomEventFilter } type messagesResp struct { - Start string `json:"start"` - StartStream string `json:"start_stream,omitempty"` // NOTSPEC: used by Cerulean, so clients can hit /messages then immediately /sync with a latest sync token - End string `json:"end,omitempty"` - Chunk []gomatrixserverlib.ClientEvent `json:"chunk"` - State []gomatrixserverlib.ClientEvent `json:"state,omitempty"` + Start string `json:"start"` + StartStream string `json:"start_stream,omitempty"` // NOTSPEC: used by Cerulean, so clients can hit /messages then immediately /sync with a latest sync token + End string `json:"end,omitempty"` + Chunk []synctypes.ClientEvent `json:"chunk"` + State []synctypes.ClientEvent `json:"state,omitempty"` } // OnIncomingMessagesRequest implements the /messages endpoint from the @@ -253,7 +254,7 @@ func OnIncomingMessagesRequest( util.GetLogger(req.Context()).WithError(err).Error("failed to apply lazy loading") return jsonerror.InternalServerError() } - res.State = append(res.State, gomatrixserverlib.HeaderedToClientEvents(membershipEvents, gomatrixserverlib.FormatAll)...) + res.State = append(res.State, synctypes.HeaderedToClientEvents(membershipEvents, synctypes.FormatAll)...) } // If we didn't return any events, set the end to an empty string, so it will be omitted @@ -291,7 +292,7 @@ func getMembershipForUser(ctx context.Context, roomID, userID string, rsAPI api. // Returns an error if there was an issue talking to the database or with the // remote homeserver. func (r *messagesReq) retrieveEvents() ( - clientEvents []gomatrixserverlib.ClientEvent, start, + clientEvents []synctypes.ClientEvent, start, end types.TopologyToken, err error, ) { // Retrieve the events from the local database. @@ -323,7 +324,7 @@ func (r *messagesReq) retrieveEvents() ( // If we didn't get any event, we don't need to proceed any further. if len(events) == 0 { - return []gomatrixserverlib.ClientEvent{}, *r.from, *r.to, nil + return []synctypes.ClientEvent{}, *r.from, *r.to, nil } // Get the position of the first and the last event in the room's topology. @@ -334,7 +335,7 @@ func (r *messagesReq) retrieveEvents() ( // only have to change it in one place, i.e. the database. start, end, err = r.getStartEnd(events) if err != nil { - return []gomatrixserverlib.ClientEvent{}, *r.from, *r.to, err + return []synctypes.ClientEvent{}, *r.from, *r.to, err } // Sort the events to ensure we send them in the right order. @@ -350,7 +351,7 @@ func (r *messagesReq) retrieveEvents() ( events = reversed(events) } if len(events) == 0 { - return []gomatrixserverlib.ClientEvent{}, *r.from, *r.to, nil + return []synctypes.ClientEvent{}, *r.from, *r.to, nil } // Apply room history visibility filter @@ -362,7 +363,7 @@ func (r *messagesReq) retrieveEvents() ( "events_before": len(events), "events_after": len(filteredEvents), }).Debug("applied history visibility (messages)") - return gomatrixserverlib.HeaderedToClientEvents(filteredEvents, gomatrixserverlib.FormatAll), start, end, err + return synctypes.HeaderedToClientEvents(filteredEvents, synctypes.FormatAll), start, end, err } func (r *messagesReq) getStartEnd(events []*gomatrixserverlib.HeaderedEvent) (start, end types.TopologyToken, err error) { diff --git a/syncapi/routing/relations.go b/syncapi/routing/relations.go index fee61b0dff..79533883f5 100644 --- a/syncapi/routing/relations.go +++ b/syncapi/routing/relations.go @@ -27,14 +27,15 @@ import ( "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/syncapi/internal" "github.com/matrix-org/dendrite/syncapi/storage" + "github.com/matrix-org/dendrite/syncapi/synctypes" "github.com/matrix-org/dendrite/syncapi/types" userapi "github.com/matrix-org/dendrite/userapi/api" ) type RelationsResponse struct { - Chunk []gomatrixserverlib.ClientEvent `json:"chunk"` - NextBatch string `json:"next_batch,omitempty"` - PrevBatch string `json:"prev_batch,omitempty"` + Chunk []synctypes.ClientEvent `json:"chunk"` + NextBatch string `json:"next_batch,omitempty"` + PrevBatch string `json:"prev_batch,omitempty"` } // nolint:gocyclo @@ -85,7 +86,7 @@ func Relations( defer sqlutil.EndTransactionWithCheck(snapshot, &succeeded, &err) res := &RelationsResponse{ - Chunk: []gomatrixserverlib.ClientEvent{}, + Chunk: []synctypes.ClientEvent{}, } var events []types.StreamEvent events, res.PrevBatch, res.NextBatch, err = snapshot.RelationsFor( @@ -108,11 +109,11 @@ func Relations( // Convert the events into client events, and optionally filter based on the event // type if it was specified. - res.Chunk = make([]gomatrixserverlib.ClientEvent, 0, len(filteredEvents)) + res.Chunk = make([]synctypes.ClientEvent, 0, len(filteredEvents)) for _, event := range filteredEvents { res.Chunk = append( res.Chunk, - gomatrixserverlib.ToClientEvent(event.Event, gomatrixserverlib.FormatAll), + synctypes.ToClientEvent(event.Event, synctypes.FormatAll), ) } diff --git a/syncapi/routing/search.go b/syncapi/routing/search.go index 69fa52942f..15cb2f9b8f 100644 --- a/syncapi/routing/search.go +++ b/syncapi/routing/search.go @@ -32,6 +32,7 @@ import ( "github.com/matrix-org/dendrite/internal/fulltext" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/syncapi/storage" + "github.com/matrix-org/dendrite/syncapi/synctypes" "github.com/matrix-org/dendrite/userapi/api" ) @@ -145,7 +146,7 @@ func Search(req *http.Request, device *api.Device, syncDB storage.Database, fts // Filter on m.room.message, as otherwise we also get events like m.reaction // which "breaks" displaying results in Element Web. types := []string{"m.room.message"} - roomFilter := &gomatrixserverlib.RoomEventFilter{ + roomFilter := &synctypes.RoomEventFilter{ Rooms: &rooms, Types: &types, } @@ -166,7 +167,7 @@ func Search(req *http.Request, device *api.Device, syncDB storage.Database, fts }) } - stateForRooms := make(map[string][]gomatrixserverlib.ClientEvent) + stateForRooms := make(map[string][]synctypes.ClientEvent) for _, event := range evs { eventsBefore, eventsAfter, err := contextEvents(ctx, snapshot, event, roomFilter, searchReq) if err != nil { @@ -204,24 +205,24 @@ func Search(req *http.Request, device *api.Device, syncDB storage.Database, fts Context: SearchContextResponse{ Start: startToken.String(), End: endToken.String(), - EventsAfter: gomatrixserverlib.HeaderedToClientEvents(eventsAfter, gomatrixserverlib.FormatSync), - EventsBefore: gomatrixserverlib.HeaderedToClientEvents(eventsBefore, gomatrixserverlib.FormatSync), + EventsAfter: synctypes.HeaderedToClientEvents(eventsAfter, synctypes.FormatSync), + EventsBefore: synctypes.HeaderedToClientEvents(eventsBefore, synctypes.FormatSync), ProfileInfo: profileInfos, }, Rank: eventScore[event.EventID()].Score, - Result: gomatrixserverlib.HeaderedToClientEvent(event, gomatrixserverlib.FormatAll), + Result: synctypes.HeaderedToClientEvent(event, synctypes.FormatAll), }) roomGroup := groups[event.RoomID()] roomGroup.Results = append(roomGroup.Results, event.EventID()) groups[event.RoomID()] = roomGroup if _, ok := stateForRooms[event.RoomID()]; searchReq.SearchCategories.RoomEvents.IncludeState && !ok { - stateFilter := gomatrixserverlib.DefaultStateFilter() + stateFilter := synctypes.DefaultStateFilter() state, err := snapshot.CurrentState(ctx, event.RoomID(), &stateFilter, nil) if err != nil { logrus.WithError(err).Error("unable to get current state") return jsonerror.InternalServerError() } - stateForRooms[event.RoomID()] = gomatrixserverlib.HeaderedToClientEvents(state, gomatrixserverlib.FormatSync) + stateForRooms[event.RoomID()] = synctypes.HeaderedToClientEvents(state, synctypes.FormatSync) } } @@ -262,7 +263,7 @@ func contextEvents( ctx context.Context, snapshot storage.DatabaseTransaction, event *gomatrixserverlib.HeaderedEvent, - roomFilter *gomatrixserverlib.RoomEventFilter, + roomFilter *synctypes.RoomEventFilter, searchReq SearchRequest, ) ([]*gomatrixserverlib.HeaderedEvent, []*gomatrixserverlib.HeaderedEvent, error) { id, _, err := snapshot.SelectContextEvent(ctx, event.RoomID(), event.EventID()) @@ -300,13 +301,13 @@ type Groupings struct { } type RoomEvents struct { - EventContext EventContext `json:"event_context"` - Filter gomatrixserverlib.RoomEventFilter `json:"filter"` - Groupings Groupings `json:"groupings"` - IncludeState bool `json:"include_state"` - Keys []string `json:"keys"` - OrderBy string `json:"order_by"` - SearchTerm string `json:"search_term"` + EventContext EventContext `json:"event_context"` + Filter synctypes.RoomEventFilter `json:"filter"` + Groupings Groupings `json:"groupings"` + IncludeState bool `json:"include_state"` + Keys []string `json:"keys"` + OrderBy string `json:"order_by"` + SearchTerm string `json:"search_term"` } type SearchCategories struct { @@ -331,17 +332,17 @@ type Groups struct { } type Result struct { - Context SearchContextResponse `json:"context"` - Rank float64 `json:"rank"` - Result gomatrixserverlib.ClientEvent `json:"result"` + Context SearchContextResponse `json:"context"` + Rank float64 `json:"rank"` + Result synctypes.ClientEvent `json:"result"` } type SearchContextResponse struct { - End string `json:"end"` - EventsAfter []gomatrixserverlib.ClientEvent `json:"events_after"` - EventsBefore []gomatrixserverlib.ClientEvent `json:"events_before"` - Start string `json:"start"` - ProfileInfo map[string]ProfileInfoResponse `json:"profile_info"` + End string `json:"end"` + EventsAfter []synctypes.ClientEvent `json:"events_after"` + EventsBefore []synctypes.ClientEvent `json:"events_before"` + Start string `json:"start"` + ProfileInfo map[string]ProfileInfoResponse `json:"profile_info"` } type ProfileInfoResponse struct { @@ -350,12 +351,12 @@ type ProfileInfoResponse struct { } type RoomEventsResponse struct { - Count int `json:"count"` - Groups Groups `json:"groups"` - Highlights []string `json:"highlights"` - NextBatch *string `json:"next_batch,omitempty"` - Results []Result `json:"results"` - State map[string][]gomatrixserverlib.ClientEvent `json:"state,omitempty"` + Count int `json:"count"` + Groups Groups `json:"groups"` + Highlights []string `json:"highlights"` + NextBatch *string `json:"next_batch,omitempty"` + Results []Result `json:"results"` + State map[string][]synctypes.ClientEvent `json:"state,omitempty"` } type SearchCategoriesResponse struct { RoomEvents RoomEventsResponse `json:"room_events"` diff --git a/syncapi/routing/search_test.go b/syncapi/routing/search_test.go index 05479300e9..716f3bb887 100644 --- a/syncapi/routing/search_test.go +++ b/syncapi/routing/search_test.go @@ -10,6 +10,7 @@ import ( "github.com/matrix-org/dendrite/internal/fulltext" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/syncapi/storage" + "github.com/matrix-org/dendrite/syncapi/synctypes" "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/dendrite/test" "github.com/matrix-org/dendrite/test/testrig" @@ -96,7 +97,7 @@ func TestSearch(t *testing.T) { SearchCategories: SearchCategories{ RoomEvents: RoomEvents{ SearchTerm: "hello", - Filter: gomatrixserverlib.RoomEventFilter{ + Filter: synctypes.RoomEventFilter{ Rooms: &roomsFilter, }, }, @@ -111,7 +112,7 @@ func TestSearch(t *testing.T) { SearchCategories: SearchCategories{ RoomEvents: RoomEvents{ SearchTerm: "hello", - Filter: gomatrixserverlib.RoomEventFilter{ + Filter: synctypes.RoomEventFilter{ Rooms: &roomsFilterUnknown, }, }, @@ -126,7 +127,7 @@ func TestSearch(t *testing.T) { SearchCategories: SearchCategories{ RoomEvents: RoomEvents{ SearchTerm: "hello", - Filter: gomatrixserverlib.RoomEventFilter{ + Filter: synctypes.RoomEventFilter{ Rooms: &roomsFilter, }, IncludeState: true, @@ -143,7 +144,7 @@ func TestSearch(t *testing.T) { SearchCategories: SearchCategories{ RoomEvents: RoomEvents{ SearchTerm: "hello", - Filter: gomatrixserverlib.RoomEventFilter{ + Filter: synctypes.RoomEventFilter{ Rooms: &roomsFilter, }, }, @@ -160,7 +161,7 @@ func TestSearch(t *testing.T) { SearchCategories: SearchCategories{ RoomEvents: RoomEvents{ SearchTerm: "hello", - Filter: gomatrixserverlib.RoomEventFilter{ + Filter: synctypes.RoomEventFilter{ Rooms: &roomsFilter, }, }, @@ -176,7 +177,7 @@ func TestSearch(t *testing.T) { SearchCategories: SearchCategories{ RoomEvents: RoomEvents{ SearchTerm: "hello", - Filter: gomatrixserverlib.RoomEventFilter{ + Filter: synctypes.RoomEventFilter{ Rooms: &roomsFilter, }, }, diff --git a/syncapi/storage/interface.go b/syncapi/storage/interface.go index 04c2020a01..38c04e8587 100644 --- a/syncapi/storage/interface.go +++ b/syncapi/storage/interface.go @@ -23,6 +23,7 @@ import ( "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/syncapi/storage/shared" + "github.com/matrix-org/dendrite/syncapi/synctypes" "github.com/matrix-org/dendrite/syncapi/types" userapi "github.com/matrix-org/dendrite/userapi/api" ) @@ -40,13 +41,13 @@ type DatabaseTransaction interface { MaxStreamPositionForPresence(ctx context.Context) (types.StreamPosition, error) MaxStreamPositionForRelations(ctx context.Context) (types.StreamPosition, error) - CurrentState(ctx context.Context, roomID string, stateFilterPart *gomatrixserverlib.StateFilter, excludeEventIDs []string) ([]*gomatrixserverlib.HeaderedEvent, error) - GetStateDeltasForFullStateSync(ctx context.Context, device *userapi.Device, r types.Range, userID string, stateFilter *gomatrixserverlib.StateFilter) ([]types.StateDelta, []string, error) - GetStateDeltas(ctx context.Context, device *userapi.Device, r types.Range, userID string, stateFilter *gomatrixserverlib.StateFilter) ([]types.StateDelta, []string, error) + CurrentState(ctx context.Context, roomID string, stateFilterPart *synctypes.StateFilter, excludeEventIDs []string) ([]*gomatrixserverlib.HeaderedEvent, error) + GetStateDeltasForFullStateSync(ctx context.Context, device *userapi.Device, r types.Range, userID string, stateFilter *synctypes.StateFilter) ([]types.StateDelta, []string, error) + GetStateDeltas(ctx context.Context, device *userapi.Device, r types.Range, userID string, stateFilter *synctypes.StateFilter) ([]types.StateDelta, []string, error) RoomIDsWithMembership(ctx context.Context, userID string, membership string) ([]string, error) MembershipCount(ctx context.Context, roomID, membership string, pos types.StreamPosition) (int, error) GetRoomSummary(ctx context.Context, roomID, userID string) (summary *types.Summary, err error) - RecentEvents(ctx context.Context, roomIDs []string, r types.Range, eventFilter *gomatrixserverlib.RoomEventFilter, chronologicalOrder bool, onlySyncEvents bool) (map[string]types.RecentEvents, error) + RecentEvents(ctx context.Context, roomIDs []string, r types.Range, eventFilter *synctypes.RoomEventFilter, chronologicalOrder bool, onlySyncEvents bool) (map[string]types.RecentEvents, error) GetBackwardTopologyPos(ctx context.Context, events []*gomatrixserverlib.HeaderedEvent) (types.TopologyToken, error) PositionInTopology(ctx context.Context, eventID string) (pos types.StreamPosition, spos types.StreamPosition, err error) InviteEventsInRange(ctx context.Context, targetUserID string, r types.Range) (map[string]*gomatrixserverlib.HeaderedEvent, map[string]*gomatrixserverlib.HeaderedEvent, types.StreamPosition, error) @@ -71,15 +72,15 @@ type DatabaseTransaction interface { // GetStateEventsForRoom fetches the state events for a given room. // Returns an empty slice if no state events could be found for this room. // Returns an error if there was an issue with the retrieval. - GetStateEventsForRoom(ctx context.Context, roomID string, stateFilterPart *gomatrixserverlib.StateFilter) (stateEvents []*gomatrixserverlib.HeaderedEvent, err error) + GetStateEventsForRoom(ctx context.Context, roomID string, stateFilterPart *synctypes.StateFilter) (stateEvents []*gomatrixserverlib.HeaderedEvent, err error) // GetAccountDataInRange returns all account data for a given user inserted or // updated between two given positions // Returns a map following the format data[roomID] = []dataTypes // If no data is retrieved, returns an empty map // If there was an issue with the retrieval, returns an error - GetAccountDataInRange(ctx context.Context, userID string, r types.Range, accountDataFilterPart *gomatrixserverlib.EventFilter) (map[string][]string, types.StreamPosition, error) + GetAccountDataInRange(ctx context.Context, userID string, r types.Range, accountDataFilterPart *synctypes.EventFilter) (map[string][]string, types.StreamPosition, error) // GetEventsInTopologicalRange retrieves all of the events on a given ordering using the given extremities and limit. If backwardsOrdering is true, the most recent event must be first, else last. - GetEventsInTopologicalRange(ctx context.Context, from, to *types.TopologyToken, roomID string, filter *gomatrixserverlib.RoomEventFilter, backwardOrdering bool) (events []types.StreamEvent, err error) + GetEventsInTopologicalRange(ctx context.Context, from, to *types.TopologyToken, roomID string, filter *synctypes.RoomEventFilter, backwardOrdering bool) (events []types.StreamEvent, err error) // EventPositionInTopology returns the depth and stream position of the given event. EventPositionInTopology(ctx context.Context, eventID string) (types.TopologyToken, error) // BackwardExtremitiesForRoom returns a map of backwards extremity event ID to a list of its prev_events. @@ -94,8 +95,8 @@ type DatabaseTransaction interface { // GetRoomReceipts gets all receipts for a given roomID GetRoomReceipts(ctx context.Context, roomIDs []string, streamPos types.StreamPosition) ([]types.OutputReceiptEvent, error) SelectContextEvent(ctx context.Context, roomID, eventID string) (int, gomatrixserverlib.HeaderedEvent, error) - SelectContextBeforeEvent(ctx context.Context, id int, roomID string, filter *gomatrixserverlib.RoomEventFilter) ([]*gomatrixserverlib.HeaderedEvent, error) - SelectContextAfterEvent(ctx context.Context, id int, roomID string, filter *gomatrixserverlib.RoomEventFilter) (int, []*gomatrixserverlib.HeaderedEvent, error) + SelectContextBeforeEvent(ctx context.Context, id int, roomID string, filter *synctypes.RoomEventFilter) ([]*gomatrixserverlib.HeaderedEvent, error) + SelectContextAfterEvent(ctx context.Context, id int, roomID string, filter *synctypes.RoomEventFilter) (int, []*gomatrixserverlib.HeaderedEvent, error) StreamToTopologicalPosition(ctx context.Context, roomID string, streamPos types.StreamPosition, backwardOrdering bool) (types.TopologyToken, error) IgnoresForUser(ctx context.Context, userID string) (*types.IgnoredUsers, error) // SelectMembershipForUser returns the membership of the user before and including the given position. If no membership can be found @@ -105,7 +106,7 @@ type DatabaseTransaction interface { // getUserUnreadNotificationCountsForRooms returns the unread notifications for the given rooms GetUserUnreadNotificationCountsForRooms(ctx context.Context, userID string, roomIDs map[string]string) (map[string]*eventutil.NotificationData, error) GetPresences(ctx context.Context, userID []string) ([]*types.PresenceInternal, error) - PresenceAfter(ctx context.Context, after types.StreamPosition, filter gomatrixserverlib.EventFilter) (map[string]*types.PresenceInternal, error) + PresenceAfter(ctx context.Context, after types.StreamPosition, filter synctypes.EventFilter) (map[string]*types.PresenceInternal, error) RelationsFor(ctx context.Context, roomID, eventID, relType, eventType string, from, to types.StreamPosition, backwards bool, limit int) (events []types.StreamEvent, prevBatch, nextBatch string, err error) } @@ -165,11 +166,11 @@ type Database interface { // GetFilter looks up the filter associated with a given local user and filter ID // and populates the target filter. Otherwise returns an error if no such filter exists // or if there was an error talking to the database. - GetFilter(ctx context.Context, target *gomatrixserverlib.Filter, localpart string, filterID string) error + GetFilter(ctx context.Context, target *synctypes.Filter, localpart string, filterID string) error // PutFilter puts the passed filter into the database. // Returns the filterID as a string. Otherwise returns an error if something // goes wrong. - PutFilter(ctx context.Context, localpart string, filter *gomatrixserverlib.Filter) (string, error) + PutFilter(ctx context.Context, localpart string, filter *synctypes.Filter) (string, error) // RedactEvent wipes an event in the database and sets the unsigned.redacted_because key to the redaction event RedactEvent(ctx context.Context, redactedEventID string, redactedBecause *gomatrixserverlib.HeaderedEvent) error // StoreReceipt stores new receipt events diff --git a/syncapi/storage/postgres/account_data_table.go b/syncapi/storage/postgres/account_data_table.go index 8147b48689..44d735bb4f 100644 --- a/syncapi/storage/postgres/account_data_table.go +++ b/syncapi/storage/postgres/account_data_table.go @@ -23,8 +23,8 @@ import ( "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/syncapi/storage/tables" + "github.com/matrix-org/dendrite/syncapi/synctypes" "github.com/matrix-org/dendrite/syncapi/types" - "github.com/matrix-org/gomatrixserverlib" ) const accountDataSchema = ` @@ -97,7 +97,7 @@ func (s *accountDataStatements) SelectAccountDataInRange( ctx context.Context, txn *sql.Tx, userID string, r types.Range, - accountDataEventFilter *gomatrixserverlib.EventFilter, + accountDataEventFilter *synctypes.EventFilter, ) (data map[string][]string, pos types.StreamPosition, err error) { data = make(map[string][]string) diff --git a/syncapi/storage/postgres/current_room_state_table.go b/syncapi/storage/postgres/current_room_state_table.go index 0d607b7c0b..b05477585e 100644 --- a/syncapi/storage/postgres/current_room_state_table.go +++ b/syncapi/storage/postgres/current_room_state_table.go @@ -26,6 +26,7 @@ import ( "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/syncapi/storage/postgres/deltas" "github.com/matrix-org/dendrite/syncapi/storage/tables" + "github.com/matrix-org/dendrite/syncapi/synctypes" "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/gomatrixserverlib" ) @@ -270,7 +271,7 @@ func (s *currentRoomStateStatements) SelectRoomIDsWithAnyMembership( // SelectCurrentState returns all the current state events for the given room. func (s *currentRoomStateStatements) SelectCurrentState( ctx context.Context, txn *sql.Tx, roomID string, - stateFilter *gomatrixserverlib.StateFilter, + stateFilter *synctypes.StateFilter, excludeEventIDs []string, ) ([]*gomatrixserverlib.HeaderedEvent, error) { stmt := sqlutil.TxStmt(txn, s.selectCurrentStateStmt) diff --git a/syncapi/storage/postgres/filter_table.go b/syncapi/storage/postgres/filter_table.go index 44c3de72ef..089382b890 100644 --- a/syncapi/storage/postgres/filter_table.go +++ b/syncapi/storage/postgres/filter_table.go @@ -21,6 +21,7 @@ import ( "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/syncapi/storage/tables" + "github.com/matrix-org/dendrite/syncapi/synctypes" "github.com/matrix-org/gomatrixserverlib" ) @@ -69,7 +70,7 @@ func NewPostgresFilterTable(db *sql.DB) (tables.Filter, error) { } func (s *filterStatements) SelectFilter( - ctx context.Context, txn *sql.Tx, target *gomatrixserverlib.Filter, localpart string, filterID string, + ctx context.Context, txn *sql.Tx, target *synctypes.Filter, localpart string, filterID string, ) error { // Retrieve filter from database (stored as canonical JSON) var filterData []byte @@ -86,7 +87,7 @@ func (s *filterStatements) SelectFilter( } func (s *filterStatements) InsertFilter( - ctx context.Context, txn *sql.Tx, filter *gomatrixserverlib.Filter, localpart string, + ctx context.Context, txn *sql.Tx, filter *synctypes.Filter, localpart string, ) (filterID string, err error) { var existingFilterID string diff --git a/syncapi/storage/postgres/filtering.go b/syncapi/storage/postgres/filtering.go index a2ca421561..39ef11086d 100644 --- a/syncapi/storage/postgres/filtering.go +++ b/syncapi/storage/postgres/filtering.go @@ -17,7 +17,7 @@ package postgres import ( "strings" - "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/dendrite/syncapi/synctypes" ) // filterConvertWildcardToSQL converts wildcards as defined in @@ -39,7 +39,7 @@ func filterConvertTypeWildcardToSQL(values *[]string) []string { } // TODO: Replace when Dendrite uses Go 1.18 -func getSendersRoomEventFilter(filter *gomatrixserverlib.RoomEventFilter) (senders []string, notSenders []string) { +func getSendersRoomEventFilter(filter *synctypes.RoomEventFilter) (senders []string, notSenders []string) { if filter.Senders != nil { senders = *filter.Senders } @@ -49,7 +49,7 @@ func getSendersRoomEventFilter(filter *gomatrixserverlib.RoomEventFilter) (sende return senders, notSenders } -func getSendersStateFilterFilter(filter *gomatrixserverlib.StateFilter) (senders []string, notSenders []string) { +func getSendersStateFilterFilter(filter *synctypes.StateFilter) (senders []string, notSenders []string) { if filter.Senders != nil { senders = *filter.Senders } diff --git a/syncapi/storage/postgres/output_room_events_table.go b/syncapi/storage/postgres/output_room_events_table.go index 59fb99aa35..3900ac3ae4 100644 --- a/syncapi/storage/postgres/output_room_events_table.go +++ b/syncapi/storage/postgres/output_room_events_table.go @@ -28,6 +28,7 @@ import ( "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/syncapi/storage/postgres/deltas" "github.com/matrix-org/dendrite/syncapi/storage/tables" + "github.com/matrix-org/dendrite/syncapi/synctypes" "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/gomatrixserverlib" ) @@ -288,7 +289,7 @@ func (s *outputRoomEventsStatements) UpdateEventJSON(ctx context.Context, txn *s // two positions, only the most recent state is returned. func (s *outputRoomEventsStatements) SelectStateInRange( ctx context.Context, txn *sql.Tx, r types.Range, - stateFilter *gomatrixserverlib.StateFilter, roomIDs []string, + stateFilter *synctypes.StateFilter, roomIDs []string, ) (map[string]map[string]bool, map[string]types.StreamEvent, error) { var rows *sql.Rows var err error @@ -433,7 +434,7 @@ func (s *outputRoomEventsStatements) InsertEvent( // from sync. func (s *outputRoomEventsStatements) SelectRecentEvents( ctx context.Context, txn *sql.Tx, - roomIDs []string, ra types.Range, eventFilter *gomatrixserverlib.RoomEventFilter, + roomIDs []string, ra types.Range, eventFilter *synctypes.RoomEventFilter, chronologicalOrder bool, onlySyncEvents bool, ) (map[string]types.RecentEvents, error) { var stmt *sql.Stmt @@ -533,7 +534,7 @@ func (s *outputRoomEventsStatements) SelectRecentEvents( // from a given position, up to a maximum of 'limit'. func (s *outputRoomEventsStatements) SelectEarlyEvents( ctx context.Context, txn *sql.Tx, - roomID string, r types.Range, eventFilter *gomatrixserverlib.RoomEventFilter, + roomID string, r types.Range, eventFilter *synctypes.RoomEventFilter, ) ([]types.StreamEvent, error) { senders, notSenders := getSendersRoomEventFilter(eventFilter) stmt := sqlutil.TxStmt(txn, s.selectEarlyEventsStmt) @@ -565,7 +566,7 @@ func (s *outputRoomEventsStatements) SelectEarlyEvents( // selectEvents returns the events for the given event IDs. If an event is // missing from the database, it will be omitted. func (s *outputRoomEventsStatements) SelectEvents( - ctx context.Context, txn *sql.Tx, eventIDs []string, filter *gomatrixserverlib.RoomEventFilter, preserveOrder bool, + ctx context.Context, txn *sql.Tx, eventIDs []string, filter *synctypes.RoomEventFilter, preserveOrder bool, ) ([]types.StreamEvent, error) { var ( stmt *sql.Stmt @@ -637,7 +638,7 @@ func (s *outputRoomEventsStatements) SelectContextEvent(ctx context.Context, txn } func (s *outputRoomEventsStatements) SelectContextBeforeEvent( - ctx context.Context, txn *sql.Tx, id int, roomID string, filter *gomatrixserverlib.RoomEventFilter, + ctx context.Context, txn *sql.Tx, id int, roomID string, filter *synctypes.RoomEventFilter, ) (evts []*gomatrixserverlib.HeaderedEvent, err error) { senders, notSenders := getSendersRoomEventFilter(filter) rows, err := sqlutil.TxStmt(txn, s.selectContextBeforeEventStmt).QueryContext( @@ -672,7 +673,7 @@ func (s *outputRoomEventsStatements) SelectContextBeforeEvent( } func (s *outputRoomEventsStatements) SelectContextAfterEvent( - ctx context.Context, txn *sql.Tx, id int, roomID string, filter *gomatrixserverlib.RoomEventFilter, + ctx context.Context, txn *sql.Tx, id int, roomID string, filter *synctypes.RoomEventFilter, ) (lastID int, evts []*gomatrixserverlib.HeaderedEvent, err error) { senders, notSenders := getSendersRoomEventFilter(filter) rows, err := sqlutil.TxStmt(txn, s.selectContextAfterEventStmt).QueryContext( diff --git a/syncapi/storage/postgres/presence_table.go b/syncapi/storage/postgres/presence_table.go index a3f7c5213e..3dba7756ce 100644 --- a/syncapi/storage/postgres/presence_table.go +++ b/syncapi/storage/postgres/presence_table.go @@ -24,6 +24,7 @@ import ( "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/syncapi/synctypes" "github.com/matrix-org/dendrite/syncapi/types" ) @@ -156,7 +157,7 @@ func (p *presenceStatements) GetMaxPresenceID(ctx context.Context, txn *sql.Tx) func (p *presenceStatements) GetPresenceAfter( ctx context.Context, txn *sql.Tx, after types.StreamPosition, - filter gomatrixserverlib.EventFilter, + filter synctypes.EventFilter, ) (presences map[string]*types.PresenceInternal, err error) { presences = make(map[string]*types.PresenceInternal) stmt := sqlutil.TxStmt(txn, p.selectPresenceAfterStmt) diff --git a/syncapi/storage/shared/storage_consumer.go b/syncapi/storage/shared/storage_consumer.go index 18802d0c47..1894a09461 100644 --- a/syncapi/storage/shared/storage_consumer.go +++ b/syncapi/storage/shared/storage_consumer.go @@ -31,6 +31,7 @@ import ( "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/syncapi/storage/tables" + "github.com/matrix-org/dendrite/syncapi/synctypes" "github.com/matrix-org/dendrite/syncapi/types" ) @@ -323,13 +324,13 @@ func (d *Database) updateRoomState( } func (d *Database) GetFilter( - ctx context.Context, target *gomatrixserverlib.Filter, localpart string, filterID string, + ctx context.Context, target *synctypes.Filter, localpart string, filterID string, ) error { return d.Filter.SelectFilter(ctx, nil, target, localpart, filterID) } func (d *Database) PutFilter( - ctx context.Context, localpart string, filter *gomatrixserverlib.Filter, + ctx context.Context, localpart string, filter *synctypes.Filter, ) (string, error) { var filterID string var err error @@ -523,10 +524,10 @@ func (d *Database) SelectContextEvent(ctx context.Context, roomID, eventID strin return d.OutputEvents.SelectContextEvent(ctx, nil, roomID, eventID) } -func (d *Database) SelectContextBeforeEvent(ctx context.Context, id int, roomID string, filter *gomatrixserverlib.RoomEventFilter) ([]*gomatrixserverlib.HeaderedEvent, error) { +func (d *Database) SelectContextBeforeEvent(ctx context.Context, id int, roomID string, filter *synctypes.RoomEventFilter) ([]*gomatrixserverlib.HeaderedEvent, error) { return d.OutputEvents.SelectContextBeforeEvent(ctx, nil, id, roomID, filter) } -func (d *Database) SelectContextAfterEvent(ctx context.Context, id int, roomID string, filter *gomatrixserverlib.RoomEventFilter) (int, []*gomatrixserverlib.HeaderedEvent, error) { +func (d *Database) SelectContextAfterEvent(ctx context.Context, id int, roomID string, filter *synctypes.RoomEventFilter) (int, []*gomatrixserverlib.HeaderedEvent, error) { return d.OutputEvents.SelectContextAfterEvent(ctx, nil, id, roomID, filter) } diff --git a/syncapi/storage/shared/storage_sync.go b/syncapi/storage/shared/storage_sync.go index 931bc9e23c..a614544b5b 100644 --- a/syncapi/storage/shared/storage_sync.go +++ b/syncapi/storage/shared/storage_sync.go @@ -10,6 +10,7 @@ import ( "github.com/tidwall/gjson" "github.com/matrix-org/dendrite/internal/eventutil" + "github.com/matrix-org/dendrite/syncapi/synctypes" "github.com/matrix-org/dendrite/syncapi/types" userapi "github.com/matrix-org/dendrite/userapi/api" ) @@ -82,7 +83,7 @@ func (d *DatabaseTransaction) MaxStreamPositionForNotificationData(ctx context.C return types.StreamPosition(id), nil } -func (d *DatabaseTransaction) CurrentState(ctx context.Context, roomID string, stateFilterPart *gomatrixserverlib.StateFilter, excludeEventIDs []string) ([]*gomatrixserverlib.HeaderedEvent, error) { +func (d *DatabaseTransaction) CurrentState(ctx context.Context, roomID string, stateFilterPart *synctypes.StateFilter, excludeEventIDs []string) ([]*gomatrixserverlib.HeaderedEvent, error) { return d.CurrentRoomState.SelectCurrentState(ctx, d.txn, roomID, stateFilterPart, excludeEventIDs) } @@ -109,7 +110,7 @@ func (d *DatabaseTransaction) GetRoomSummary(ctx context.Context, roomID, userID summary.JoinedMemberCount = &joinCount // Get the room name and canonical alias, if any - filter := gomatrixserverlib.DefaultStateFilter() + filter := synctypes.DefaultStateFilter() filterTypes := []string{gomatrixserverlib.MRoomName, gomatrixserverlib.MRoomCanonicalAlias} filterRooms := []string{roomID} @@ -151,7 +152,7 @@ func (d *DatabaseTransaction) GetRoomSummary(ctx context.Context, roomID, userID return summary, nil } -func (d *DatabaseTransaction) RecentEvents(ctx context.Context, roomIDs []string, r types.Range, eventFilter *gomatrixserverlib.RoomEventFilter, chronologicalOrder bool, onlySyncEvents bool) (map[string]types.RecentEvents, error) { +func (d *DatabaseTransaction) RecentEvents(ctx context.Context, roomIDs []string, r types.Range, eventFilter *synctypes.RoomEventFilter, chronologicalOrder bool, onlySyncEvents bool) (map[string]types.RecentEvents, error) { return d.OutputEvents.SelectRecentEvents(ctx, d.txn, roomIDs, r, eventFilter, chronologicalOrder, onlySyncEvents) } @@ -210,7 +211,7 @@ func (d *DatabaseTransaction) GetStateEvent( } func (d *DatabaseTransaction) GetStateEventsForRoom( - ctx context.Context, roomID string, stateFilter *gomatrixserverlib.StateFilter, + ctx context.Context, roomID string, stateFilter *synctypes.StateFilter, ) (stateEvents []*gomatrixserverlib.HeaderedEvent, err error) { stateEvents, err = d.CurrentRoomState.SelectCurrentState(ctx, d.txn, roomID, stateFilter, nil) return @@ -223,7 +224,7 @@ func (d *DatabaseTransaction) GetStateEventsForRoom( // If there was an issue with the retrieval, returns an error func (d *DatabaseTransaction) GetAccountDataInRange( ctx context.Context, userID string, r types.Range, - accountDataFilterPart *gomatrixserverlib.EventFilter, + accountDataFilterPart *synctypes.EventFilter, ) (map[string][]string, types.StreamPosition, error) { return d.AccountData.SelectAccountDataInRange(ctx, d.txn, userID, r, accountDataFilterPart) } @@ -232,7 +233,7 @@ func (d *DatabaseTransaction) GetEventsInTopologicalRange( ctx context.Context, from, to *types.TopologyToken, roomID string, - filter *gomatrixserverlib.RoomEventFilter, + filter *synctypes.RoomEventFilter, backwardOrdering bool, ) (events []types.StreamEvent, err error) { var minDepth, maxDepth, maxStreamPosForMaxDepth types.StreamPosition @@ -323,7 +324,7 @@ func (d *DatabaseTransaction) GetBackwardTopologyPos( func (d *DatabaseTransaction) GetStateDeltas( ctx context.Context, device *userapi.Device, r types.Range, userID string, - stateFilter *gomatrixserverlib.StateFilter, + stateFilter *synctypes.StateFilter, ) (deltas []types.StateDelta, joinedRoomsIDs []string, err error) { // Implement membership change algorithm: https://github.com/matrix-org/synapse/blob/v0.19.3/synapse/handlers/sync.py#L821 // - Get membership list changes for this user in this sync response @@ -488,7 +489,7 @@ func (d *DatabaseTransaction) GetStateDeltas( func (d *DatabaseTransaction) GetStateDeltasForFullStateSync( ctx context.Context, device *userapi.Device, r types.Range, userID string, - stateFilter *gomatrixserverlib.StateFilter, + stateFilter *synctypes.StateFilter, ) ([]types.StateDelta, []string, error) { // Look up all memberships for the user. We only care about rooms that a // user has ever interacted with — joined to, kicked/banned from, left. @@ -597,7 +598,7 @@ func (d *DatabaseTransaction) GetStateDeltasForFullStateSync( func (d *DatabaseTransaction) currentStateStreamEventsForRoom( ctx context.Context, roomID string, - stateFilter *gomatrixserverlib.StateFilter, + stateFilter *synctypes.StateFilter, ) ([]types.StreamEvent, error) { allState, err := d.CurrentRoomState.SelectCurrentState(ctx, d.txn, roomID, stateFilter, nil) if err != nil { @@ -647,7 +648,7 @@ func (d *DatabaseTransaction) GetPresences(ctx context.Context, userIDs []string return d.Presence.GetPresenceForUsers(ctx, d.txn, userIDs) } -func (d *DatabaseTransaction) PresenceAfter(ctx context.Context, after types.StreamPosition, filter gomatrixserverlib.EventFilter) (map[string]*types.PresenceInternal, error) { +func (d *DatabaseTransaction) PresenceAfter(ctx context.Context, after types.StreamPosition, filter synctypes.EventFilter) (map[string]*types.PresenceInternal, error) { return d.Presence.GetPresenceAfter(ctx, d.txn, after, filter) } @@ -707,7 +708,7 @@ func (d *DatabaseTransaction) MaxStreamPositionForRelations(ctx context.Context) return types.StreamPosition(id), err } -func isStatefilterEmpty(filter *gomatrixserverlib.StateFilter) bool { +func isStatefilterEmpty(filter *synctypes.StateFilter) bool { if filter == nil { return true } diff --git a/syncapi/storage/shared/storage_sync_test.go b/syncapi/storage/shared/storage_sync_test.go index c56720db72..4468a7728c 100644 --- a/syncapi/storage/shared/storage_sync_test.go +++ b/syncapi/storage/shared/storage_sync_test.go @@ -3,7 +3,7 @@ package shared import ( "testing" - "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/dendrite/syncapi/synctypes" ) func Test_isStatefilterEmpty(t *testing.T) { @@ -12,7 +12,7 @@ func Test_isStatefilterEmpty(t *testing.T) { tests := []struct { name string - filter *gomatrixserverlib.StateFilter + filter *synctypes.StateFilter want bool }{ { @@ -22,42 +22,42 @@ func Test_isStatefilterEmpty(t *testing.T) { }, { name: "Empty filter is empty", - filter: &gomatrixserverlib.StateFilter{}, + filter: &synctypes.StateFilter{}, want: true, }, { name: "NotTypes is set", - filter: &gomatrixserverlib.StateFilter{ + filter: &synctypes.StateFilter{ NotTypes: &filterSet, }, }, { name: "Types is set", - filter: &gomatrixserverlib.StateFilter{ + filter: &synctypes.StateFilter{ Types: &filterSet, }, }, { name: "Senders is set", - filter: &gomatrixserverlib.StateFilter{ + filter: &synctypes.StateFilter{ Senders: &filterSet, }, }, { name: "NotSenders is set", - filter: &gomatrixserverlib.StateFilter{ + filter: &synctypes.StateFilter{ NotSenders: &filterSet, }, }, { name: "NotRooms is set", - filter: &gomatrixserverlib.StateFilter{ + filter: &synctypes.StateFilter{ NotRooms: &filterSet, }, }, { name: "ContainsURL is set", - filter: &gomatrixserverlib.StateFilter{ + filter: &synctypes.StateFilter{ ContainsURL: &boolValue, }, }, diff --git a/syncapi/storage/sqlite3/account_data_table.go b/syncapi/storage/sqlite3/account_data_table.go index eaf5223409..b49c2f7015 100644 --- a/syncapi/storage/sqlite3/account_data_table.go +++ b/syncapi/storage/sqlite3/account_data_table.go @@ -22,8 +22,8 @@ import ( "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/syncapi/storage/tables" + "github.com/matrix-org/dendrite/syncapi/synctypes" "github.com/matrix-org/dendrite/syncapi/types" - "github.com/matrix-org/gomatrixserverlib" ) const accountDataSchema = ` @@ -89,7 +89,7 @@ func (s *accountDataStatements) SelectAccountDataInRange( ctx context.Context, txn *sql.Tx, userID string, r types.Range, - filter *gomatrixserverlib.EventFilter, + filter *synctypes.EventFilter, ) (data map[string][]string, pos types.StreamPosition, err error) { data = make(map[string][]string) stmt, params, err := prepareWithFilters( diff --git a/syncapi/storage/sqlite3/current_room_state_table.go b/syncapi/storage/sqlite3/current_room_state_table.go index 35b746c5c6..c681933d5d 100644 --- a/syncapi/storage/sqlite3/current_room_state_table.go +++ b/syncapi/storage/sqlite3/current_room_state_table.go @@ -27,6 +27,7 @@ import ( "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/syncapi/storage/sqlite3/deltas" "github.com/matrix-org/dendrite/syncapi/storage/tables" + "github.com/matrix-org/dendrite/syncapi/synctypes" "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/gomatrixserverlib" ) @@ -264,7 +265,7 @@ func (s *currentRoomStateStatements) SelectRoomIDsWithAnyMembership( // CurrentState returns all the current state events for the given room. func (s *currentRoomStateStatements) SelectCurrentState( ctx context.Context, txn *sql.Tx, roomID string, - stateFilter *gomatrixserverlib.StateFilter, + stateFilter *synctypes.StateFilter, excludeEventIDs []string, ) ([]*gomatrixserverlib.HeaderedEvent, error) { // We're going to query members later, so remove them from this request diff --git a/syncapi/storage/sqlite3/filter_table.go b/syncapi/storage/sqlite3/filter_table.go index 0132a7664a..8da4f299cb 100644 --- a/syncapi/storage/sqlite3/filter_table.go +++ b/syncapi/storage/sqlite3/filter_table.go @@ -22,6 +22,7 @@ import ( "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/syncapi/storage/tables" + "github.com/matrix-org/dendrite/syncapi/synctypes" "github.com/matrix-org/gomatrixserverlib" ) @@ -73,7 +74,7 @@ func NewSqliteFilterTable(db *sql.DB) (tables.Filter, error) { } func (s *filterStatements) SelectFilter( - ctx context.Context, txn *sql.Tx, target *gomatrixserverlib.Filter, localpart string, filterID string, + ctx context.Context, txn *sql.Tx, target *synctypes.Filter, localpart string, filterID string, ) error { // Retrieve filter from database (stored as canonical JSON) var filterData []byte @@ -90,7 +91,7 @@ func (s *filterStatements) SelectFilter( } func (s *filterStatements) InsertFilter( - ctx context.Context, txn *sql.Tx, filter *gomatrixserverlib.Filter, localpart string, + ctx context.Context, txn *sql.Tx, filter *synctypes.Filter, localpart string, ) (filterID string, err error) { var existingFilterID string diff --git a/syncapi/storage/sqlite3/output_room_events_table.go b/syncapi/storage/sqlite3/output_room_events_table.go index 23bc68a412..33ca687df1 100644 --- a/syncapi/storage/sqlite3/output_room_events_table.go +++ b/syncapi/storage/sqlite3/output_room_events_table.go @@ -27,6 +27,7 @@ import ( "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/syncapi/storage/sqlite3/deltas" "github.com/matrix-org/dendrite/syncapi/storage/tables" + "github.com/matrix-org/dendrite/syncapi/synctypes" "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/gomatrixserverlib" @@ -186,7 +187,7 @@ func (s *outputRoomEventsStatements) UpdateEventJSON(ctx context.Context, txn *s // two positions, only the most recent state is returned. func (s *outputRoomEventsStatements) SelectStateInRange( ctx context.Context, txn *sql.Tx, r types.Range, - stateFilter *gomatrixserverlib.StateFilter, roomIDs []string, + stateFilter *synctypes.StateFilter, roomIDs []string, ) (map[string]map[string]bool, map[string]types.StreamEvent, error) { stmtSQL := strings.Replace(selectStateInRangeSQL, "($3)", sqlutil.QueryVariadicOffset(len(roomIDs), 2), 1) inputParams := []interface{}{ @@ -368,7 +369,7 @@ func (s *outputRoomEventsStatements) InsertEvent( func (s *outputRoomEventsStatements) SelectRecentEvents( ctx context.Context, txn *sql.Tx, - roomIDs []string, r types.Range, eventFilter *gomatrixserverlib.RoomEventFilter, + roomIDs []string, r types.Range, eventFilter *synctypes.RoomEventFilter, chronologicalOrder bool, onlySyncEvents bool, ) (map[string]types.RecentEvents, error) { var query string @@ -431,7 +432,7 @@ func (s *outputRoomEventsStatements) SelectRecentEvents( func (s *outputRoomEventsStatements) SelectEarlyEvents( ctx context.Context, txn *sql.Tx, - roomID string, r types.Range, eventFilter *gomatrixserverlib.RoomEventFilter, + roomID string, r types.Range, eventFilter *synctypes.RoomEventFilter, ) ([]types.StreamEvent, error) { stmt, params, err := prepareWithFilters( s.db, txn, selectEarlyEventsSQL, @@ -468,7 +469,7 @@ func (s *outputRoomEventsStatements) SelectEarlyEvents( // selectEvents returns the events for the given event IDs. If an event is // missing from the database, it will be omitted. func (s *outputRoomEventsStatements) SelectEvents( - ctx context.Context, txn *sql.Tx, eventIDs []string, filter *gomatrixserverlib.RoomEventFilter, preserveOrder bool, + ctx context.Context, txn *sql.Tx, eventIDs []string, filter *synctypes.RoomEventFilter, preserveOrder bool, ) ([]types.StreamEvent, error) { iEventIDs := make([]interface{}, len(eventIDs)) for i := range eventIDs { @@ -477,7 +478,7 @@ func (s *outputRoomEventsStatements) SelectEvents( selectSQL := strings.Replace(selectEventsSQL, "($1)", sqlutil.QueryVariadic(len(eventIDs)), 1) if filter == nil { - filter = &gomatrixserverlib.RoomEventFilter{Limit: 20} + filter = &synctypes.RoomEventFilter{Limit: 20} } stmt, params, err := prepareWithFilters( s.db, txn, selectSQL, iEventIDs, @@ -581,7 +582,7 @@ func (s *outputRoomEventsStatements) SelectContextEvent( } func (s *outputRoomEventsStatements) SelectContextBeforeEvent( - ctx context.Context, txn *sql.Tx, id int, roomID string, filter *gomatrixserverlib.RoomEventFilter, + ctx context.Context, txn *sql.Tx, id int, roomID string, filter *synctypes.RoomEventFilter, ) (evts []*gomatrixserverlib.HeaderedEvent, err error) { stmt, params, err := prepareWithFilters( s.db, txn, selectContextBeforeEventSQL, @@ -623,7 +624,7 @@ func (s *outputRoomEventsStatements) SelectContextBeforeEvent( } func (s *outputRoomEventsStatements) SelectContextAfterEvent( - ctx context.Context, txn *sql.Tx, id int, roomID string, filter *gomatrixserverlib.RoomEventFilter, + ctx context.Context, txn *sql.Tx, id int, roomID string, filter *synctypes.RoomEventFilter, ) (lastID int, evts []*gomatrixserverlib.HeaderedEvent, err error) { stmt, params, err := prepareWithFilters( s.db, txn, selectContextAfterEventSQL, diff --git a/syncapi/storage/sqlite3/presence_table.go b/syncapi/storage/sqlite3/presence_table.go index 7641de92f0..5f76b37003 100644 --- a/syncapi/storage/sqlite3/presence_table.go +++ b/syncapi/storage/sqlite3/presence_table.go @@ -24,6 +24,7 @@ import ( "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/syncapi/synctypes" "github.com/matrix-org/dendrite/syncapi/types" ) @@ -180,7 +181,7 @@ func (p *presenceStatements) GetMaxPresenceID(ctx context.Context, txn *sql.Tx) // GetPresenceAfter returns the changes presences after a given stream id func (p *presenceStatements) GetPresenceAfter( ctx context.Context, txn *sql.Tx, - after types.StreamPosition, filter gomatrixserverlib.EventFilter, + after types.StreamPosition, filter synctypes.EventFilter, ) (presences map[string]*types.PresenceInternal, err error) { presences = make(map[string]*types.PresenceInternal) stmt := sqlutil.TxStmt(txn, p.selectPresenceAfterStmt) diff --git a/syncapi/storage/storage_test.go b/syncapi/storage/storage_test.go index e81a341f19..fa3f54c0ec 100644 --- a/syncapi/storage/storage_test.go +++ b/syncapi/storage/storage_test.go @@ -12,6 +12,7 @@ import ( "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/syncapi/storage" + "github.com/matrix-org/dendrite/syncapi/synctypes" "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/dendrite/test" "github.com/matrix-org/gomatrixserverlib" @@ -153,7 +154,7 @@ func TestRecentEventsPDU(t *testing.T) { for i := range testCases { tc := testCases[i] t.Run(tc.Name, func(st *testing.T) { - var filter gomatrixserverlib.RoomEventFilter + var filter synctypes.RoomEventFilter var gotEvents map[string]types.RecentEvents var limited bool filter.Limit = tc.Limit @@ -206,7 +207,7 @@ func TestGetEventsInRangeWithTopologyToken(t *testing.T) { to := types.TopologyToken{} // backpaginate 5 messages starting at the latest position. - filter := &gomatrixserverlib.RoomEventFilter{Limit: 5} + filter := &synctypes.RoomEventFilter{Limit: 5} paginatedEvents, err := snapshot.GetEventsInTopologicalRange(ctx, &from, &to, r.ID, filter, true) if err != nil { t.Fatalf("GetEventsInTopologicalRange returned an error: %s", err) @@ -932,7 +933,7 @@ func TestRecentEvents(t *testing.T) { } test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { - filter := gomatrixserverlib.DefaultRoomEventFilter() + filter := synctypes.DefaultRoomEventFilter() db, close := MustCreateDatabase(t, dbType) t.Cleanup(close) diff --git a/syncapi/storage/tables/current_room_state_test.go b/syncapi/storage/tables/current_room_state_test.go index c7af4f9776..5fe06c3cec 100644 --- a/syncapi/storage/tables/current_room_state_test.go +++ b/syncapi/storage/tables/current_room_state_test.go @@ -11,6 +11,7 @@ import ( "github.com/matrix-org/dendrite/syncapi/storage/postgres" "github.com/matrix-org/dendrite/syncapi/storage/sqlite3" "github.com/matrix-org/dendrite/syncapi/storage/tables" + "github.com/matrix-org/dendrite/syncapi/synctypes" "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/dendrite/test" "github.com/matrix-org/gomatrixserverlib" @@ -94,7 +95,7 @@ func TestCurrentRoomStateTable(t *testing.T) { func testCurrentState(t *testing.T, ctx context.Context, txn *sql.Tx, tab tables.CurrentRoomState, room *test.Room) { t.Run("test currentState", func(t *testing.T) { // returns the complete state of the room with a default filter - filter := gomatrixserverlib.DefaultStateFilter() + filter := synctypes.DefaultStateFilter() evs, err := tab.SelectCurrentState(ctx, txn, room.ID, &filter, nil) if err != nil { t.Fatal(err) diff --git a/syncapi/storage/tables/interface.go b/syncapi/storage/tables/interface.go index 727d6bf2c3..b710e60a45 100644 --- a/syncapi/storage/tables/interface.go +++ b/syncapi/storage/tables/interface.go @@ -22,13 +22,14 @@ import ( "github.com/matrix-org/dendrite/internal/eventutil" "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/syncapi/synctypes" "github.com/matrix-org/dendrite/syncapi/types" ) type AccountData interface { InsertAccountData(ctx context.Context, txn *sql.Tx, userID, roomID, dataType string) (pos types.StreamPosition, err error) // SelectAccountDataInRange returns a map of room ID to a list of `dataType`. - SelectAccountDataInRange(ctx context.Context, txn *sql.Tx, userID string, r types.Range, accountDataEventFilter *gomatrixserverlib.EventFilter) (data map[string][]string, pos types.StreamPosition, err error) + SelectAccountDataInRange(ctx context.Context, txn *sql.Tx, userID string, r types.Range, accountDataEventFilter *synctypes.EventFilter) (data map[string][]string, pos types.StreamPosition, err error) SelectMaxAccountDataID(ctx context.Context, txn *sql.Tx) (id int64, err error) } @@ -53,7 +54,7 @@ type Peeks interface { } type Events interface { - SelectStateInRange(ctx context.Context, txn *sql.Tx, r types.Range, stateFilter *gomatrixserverlib.StateFilter, roomIDs []string) (map[string]map[string]bool, map[string]types.StreamEvent, error) + SelectStateInRange(ctx context.Context, txn *sql.Tx, r types.Range, stateFilter *synctypes.StateFilter, roomIDs []string) (map[string]map[string]bool, map[string]types.StreamEvent, error) SelectMaxEventID(ctx context.Context, txn *sql.Tx) (id int64, err error) InsertEvent( ctx context.Context, txn *sql.Tx, @@ -66,17 +67,17 @@ type Events interface { // SelectRecentEvents returns events between the two stream positions: exclusive of low and inclusive of high. // If onlySyncEvents has a value of true, only returns the events that aren't marked as to exclude from sync. // Returns up to `limit` events. Returns `limited=true` if there are more events in this range but we hit the `limit`. - SelectRecentEvents(ctx context.Context, txn *sql.Tx, roomIDs []string, r types.Range, eventFilter *gomatrixserverlib.RoomEventFilter, chronologicalOrder bool, onlySyncEvents bool) (map[string]types.RecentEvents, error) + SelectRecentEvents(ctx context.Context, txn *sql.Tx, roomIDs []string, r types.Range, eventFilter *synctypes.RoomEventFilter, chronologicalOrder bool, onlySyncEvents bool) (map[string]types.RecentEvents, error) // SelectEarlyEvents returns the earliest events in the given room. - SelectEarlyEvents(ctx context.Context, txn *sql.Tx, roomID string, r types.Range, eventFilter *gomatrixserverlib.RoomEventFilter) ([]types.StreamEvent, error) - SelectEvents(ctx context.Context, txn *sql.Tx, eventIDs []string, filter *gomatrixserverlib.RoomEventFilter, preserveOrder bool) ([]types.StreamEvent, error) + SelectEarlyEvents(ctx context.Context, txn *sql.Tx, roomID string, r types.Range, eventFilter *synctypes.RoomEventFilter) ([]types.StreamEvent, error) + SelectEvents(ctx context.Context, txn *sql.Tx, eventIDs []string, filter *synctypes.RoomEventFilter, preserveOrder bool) ([]types.StreamEvent, error) UpdateEventJSON(ctx context.Context, txn *sql.Tx, event *gomatrixserverlib.HeaderedEvent) error // DeleteEventsForRoom removes all event information for a room. This should only be done when removing the room entirely. DeleteEventsForRoom(ctx context.Context, txn *sql.Tx, roomID string) (err error) SelectContextEvent(ctx context.Context, txn *sql.Tx, roomID, eventID string) (int, gomatrixserverlib.HeaderedEvent, error) - SelectContextBeforeEvent(ctx context.Context, txn *sql.Tx, id int, roomID string, filter *gomatrixserverlib.RoomEventFilter) ([]*gomatrixserverlib.HeaderedEvent, error) - SelectContextAfterEvent(ctx context.Context, txn *sql.Tx, id int, roomID string, filter *gomatrixserverlib.RoomEventFilter) (int, []*gomatrixserverlib.HeaderedEvent, error) + SelectContextBeforeEvent(ctx context.Context, txn *sql.Tx, id int, roomID string, filter *synctypes.RoomEventFilter) ([]*gomatrixserverlib.HeaderedEvent, error) + SelectContextAfterEvent(ctx context.Context, txn *sql.Tx, id int, roomID string, filter *synctypes.RoomEventFilter) (int, []*gomatrixserverlib.HeaderedEvent, error) PurgeEvents(ctx context.Context, txn *sql.Tx, roomID string) error ReIndex(ctx context.Context, txn *sql.Tx, limit, offset int64, types []string) (map[int64]gomatrixserverlib.HeaderedEvent, error) @@ -107,7 +108,7 @@ type CurrentRoomState interface { DeleteRoomStateByEventID(ctx context.Context, txn *sql.Tx, eventID string) error DeleteRoomStateForRoom(ctx context.Context, txn *sql.Tx, roomID string) error // SelectCurrentState returns all the current state events for the given room. - SelectCurrentState(ctx context.Context, txn *sql.Tx, roomID string, stateFilter *gomatrixserverlib.StateFilter, excludeEventIDs []string) ([]*gomatrixserverlib.HeaderedEvent, error) + SelectCurrentState(ctx context.Context, txn *sql.Tx, roomID string, stateFilter *synctypes.StateFilter, excludeEventIDs []string) ([]*gomatrixserverlib.HeaderedEvent, error) // SelectRoomIDsWithMembership returns the list of room IDs which have the given user in the given membership state. SelectRoomIDsWithMembership(ctx context.Context, txn *sql.Tx, userID string, membership string) ([]string, error) // SelectRoomIDsWithAnyMembership returns a map of all memberships for the given user. @@ -179,8 +180,8 @@ type SendToDevice interface { } type Filter interface { - SelectFilter(ctx context.Context, txn *sql.Tx, target *gomatrixserverlib.Filter, localpart string, filterID string) error - InsertFilter(ctx context.Context, txn *sql.Tx, filter *gomatrixserverlib.Filter, localpart string) (filterID string, err error) + SelectFilter(ctx context.Context, txn *sql.Tx, target *synctypes.Filter, localpart string, filterID string) error + InsertFilter(ctx context.Context, txn *sql.Tx, filter *synctypes.Filter, localpart string) (filterID string, err error) } type Receipts interface { @@ -218,7 +219,7 @@ type Presence interface { UpsertPresence(ctx context.Context, txn *sql.Tx, userID string, statusMsg *string, presence types.Presence, lastActiveTS gomatrixserverlib.Timestamp, fromSync bool) (pos types.StreamPosition, err error) GetPresenceForUsers(ctx context.Context, txn *sql.Tx, userIDs []string) (presence []*types.PresenceInternal, err error) GetMaxPresenceID(ctx context.Context, txn *sql.Tx) (pos types.StreamPosition, err error) - GetPresenceAfter(ctx context.Context, txn *sql.Tx, after types.StreamPosition, filter gomatrixserverlib.EventFilter) (presences map[string]*types.PresenceInternal, err error) + GetPresenceAfter(ctx context.Context, txn *sql.Tx, after types.StreamPosition, filter synctypes.EventFilter) (presences map[string]*types.PresenceInternal, err error) } type Relations interface { diff --git a/syncapi/storage/tables/output_room_events_test.go b/syncapi/storage/tables/output_room_events_test.go index bdb17ae200..c0d4511115 100644 --- a/syncapi/storage/tables/output_room_events_test.go +++ b/syncapi/storage/tables/output_room_events_test.go @@ -12,6 +12,7 @@ import ( "github.com/matrix-org/dendrite/syncapi/storage/postgres" "github.com/matrix-org/dendrite/syncapi/storage/sqlite3" "github.com/matrix-org/dendrite/syncapi/storage/tables" + "github.com/matrix-org/dendrite/syncapi/synctypes" "github.com/matrix-org/dendrite/test" "github.com/matrix-org/gomatrixserverlib" ) @@ -84,7 +85,7 @@ func TestOutputRoomEventsTable(t *testing.T) { } wantEventID := []string{urlEv.EventID()} t := true - gotEvents, err = tab.SelectEvents(ctx, txn, wantEventID, &gomatrixserverlib.RoomEventFilter{Limit: 1, ContainsURL: &t}, true) + gotEvents, err = tab.SelectEvents(ctx, txn, wantEventID, &synctypes.RoomEventFilter{Limit: 1, ContainsURL: &t}, true) if err != nil { return fmt.Errorf("failed to SelectEvents: %s", err) } diff --git a/syncapi/storage/tables/presence_table_test.go b/syncapi/storage/tables/presence_table_test.go index dce0c695ac..cb7a4dee99 100644 --- a/syncapi/storage/tables/presence_table_test.go +++ b/syncapi/storage/tables/presence_table_test.go @@ -14,6 +14,7 @@ import ( "github.com/matrix-org/dendrite/syncapi/storage/postgres" "github.com/matrix-org/dendrite/syncapi/storage/sqlite3" "github.com/matrix-org/dendrite/syncapi/storage/tables" + "github.com/matrix-org/dendrite/syncapi/synctypes" "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/dendrite/test" ) @@ -96,7 +97,7 @@ func TestPresence(t *testing.T) { } // This should return only Bobs status - presences, err := tab.GetPresenceAfter(ctx, txn, maxPos, gomatrixserverlib.EventFilter{Limit: 10}) + presences, err := tab.GetPresenceAfter(ctx, txn, maxPos, synctypes.EventFilter{Limit: 10}) if err != nil { t.Error(err) } diff --git a/syncapi/streams/stream_accountdata.go b/syncapi/streams/stream_accountdata.go index 3593a6563c..22953b8c1c 100644 --- a/syncapi/streams/stream_accountdata.go +++ b/syncapi/streams/stream_accountdata.go @@ -6,6 +6,7 @@ import ( "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/dendrite/syncapi/storage" + "github.com/matrix-org/dendrite/syncapi/synctypes" "github.com/matrix-org/dendrite/syncapi/types" userapi "github.com/matrix-org/dendrite/userapi/api" ) @@ -82,7 +83,7 @@ func (p *AccountDataStreamProvider) IncrementalSync( if globalData, ok := dataRes.GlobalAccountData[dataType]; ok { req.Response.AccountData.Events = append( req.Response.AccountData.Events, - gomatrixserverlib.ClientEvent{ + synctypes.ClientEvent{ Type: dataType, Content: gomatrixserverlib.RawJSON(globalData), }, @@ -96,7 +97,7 @@ func (p *AccountDataStreamProvider) IncrementalSync( } joinData.AccountData.Events = append( joinData.AccountData.Events, - gomatrixserverlib.ClientEvent{ + synctypes.ClientEvent{ Type: dataType, Content: gomatrixserverlib.RawJSON(roomData), }, diff --git a/syncapi/streams/stream_invite.go b/syncapi/streams/stream_invite.go index e4de30e1c5..a4414f3154 100644 --- a/syncapi/streams/stream_invite.go +++ b/syncapi/streams/stream_invite.go @@ -11,6 +11,7 @@ import ( "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/dendrite/syncapi/storage" + "github.com/matrix-org/dendrite/syncapi/synctypes" "github.com/matrix-org/dendrite/syncapi/types" ) @@ -85,7 +86,7 @@ func (p *InviteStreamProvider) IncrementalSync( lr := types.NewLeaveResponse() h := sha256.Sum256(append([]byte(roomID), []byte(strconv.FormatInt(int64(to), 10))...)) - lr.Timeline.Events = append(lr.Timeline.Events, gomatrixserverlib.ClientEvent{ + lr.Timeline.Events = append(lr.Timeline.Events, synctypes.ClientEvent{ // fake event ID which muxes in the to position EventID: "$" + base64.RawURLEncoding.EncodeToString(h[:]), OriginServerTS: gomatrixserverlib.AsTimestamp(time.Now()), diff --git a/syncapi/streams/stream_pdu.go b/syncapi/streams/stream_pdu.go index 6af25c0288..e29e29f7a3 100644 --- a/syncapi/streams/stream_pdu.go +++ b/syncapi/streams/stream_pdu.go @@ -10,6 +10,7 @@ import ( roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/syncapi/internal" "github.com/matrix-org/dendrite/syncapi/storage" + "github.com/matrix-org/dendrite/syncapi/synctypes" "github.com/matrix-org/dendrite/syncapi/types" userapi "github.com/matrix-org/dendrite/userapi/api" @@ -242,8 +243,8 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse( device *userapi.Device, r types.Range, delta types.StateDelta, - eventFilter *gomatrixserverlib.RoomEventFilter, - stateFilter *gomatrixserverlib.StateFilter, + eventFilter *synctypes.RoomEventFilter, + stateFilter *synctypes.StateFilter, req *types.SyncRequest, ) (types.StreamPosition, error) { var err error @@ -365,20 +366,20 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse( } } jr.Timeline.PrevBatch = &prevBatch - jr.Timeline.Events = gomatrixserverlib.HeaderedToClientEvents(events, gomatrixserverlib.FormatSync) + jr.Timeline.Events = synctypes.HeaderedToClientEvents(events, synctypes.FormatSync) // If we are limited by the filter AND the history visibility filter // didn't "remove" events, return that the response is limited. jr.Timeline.Limited = (limited && len(events) == len(recentEvents)) || delta.NewlyJoined - jr.State.Events = gomatrixserverlib.HeaderedToClientEvents(delta.StateEvents, gomatrixserverlib.FormatSync) + jr.State.Events = synctypes.HeaderedToClientEvents(delta.StateEvents, synctypes.FormatSync) req.Response.Rooms.Join[delta.RoomID] = jr case gomatrixserverlib.Peek: jr := types.NewJoinResponse() jr.Timeline.PrevBatch = &prevBatch // TODO: Apply history visibility on peeked rooms - jr.Timeline.Events = gomatrixserverlib.HeaderedToClientEvents(recentEvents, gomatrixserverlib.FormatSync) + jr.Timeline.Events = synctypes.HeaderedToClientEvents(recentEvents, synctypes.FormatSync) jr.Timeline.Limited = limited - jr.State.Events = gomatrixserverlib.HeaderedToClientEvents(delta.StateEvents, gomatrixserverlib.FormatSync) + jr.State.Events = synctypes.HeaderedToClientEvents(delta.StateEvents, synctypes.FormatSync) req.Response.Rooms.Peek[delta.RoomID] = jr case gomatrixserverlib.Leave: @@ -387,11 +388,11 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse( case gomatrixserverlib.Ban: lr := types.NewLeaveResponse() lr.Timeline.PrevBatch = &prevBatch - lr.Timeline.Events = gomatrixserverlib.HeaderedToClientEvents(events, gomatrixserverlib.FormatSync) + lr.Timeline.Events = synctypes.HeaderedToClientEvents(events, synctypes.FormatSync) // If we are limited by the filter AND the history visibility filter // didn't "remove" events, return that the response is limited. lr.Timeline.Limited = limited && len(events) == len(recentEvents) - lr.State.Events = gomatrixserverlib.HeaderedToClientEvents(delta.StateEvents, gomatrixserverlib.FormatSync) + lr.State.Events = synctypes.HeaderedToClientEvents(delta.StateEvents, synctypes.FormatSync) req.Response.Rooms.Leave[delta.RoomID] = lr } @@ -420,7 +421,7 @@ func applyHistoryVisibilityFilter( // Only get the state again if there are state events in the timeline if len(stateTypes) > 0 { - filter := gomatrixserverlib.DefaultStateFilter() + filter := synctypes.DefaultStateFilter() filter.Types = &stateTypes filter.Senders = &senders stateEvents, err := snapshot.CurrentState(ctx, roomID, &filter, nil) @@ -451,7 +452,7 @@ func (p *PDUStreamProvider) getJoinResponseForCompleteSync( ctx context.Context, snapshot storage.DatabaseTransaction, roomID string, - stateFilter *gomatrixserverlib.StateFilter, + stateFilter *synctypes.StateFilter, wantFullState bool, device *userapi.Device, isPeek bool, @@ -541,17 +542,17 @@ func (p *PDUStreamProvider) getJoinResponseForCompleteSync( } jr.Timeline.PrevBatch = prevBatch - jr.Timeline.Events = gomatrixserverlib.HeaderedToClientEvents(events, gomatrixserverlib.FormatSync) + jr.Timeline.Events = synctypes.HeaderedToClientEvents(events, synctypes.FormatSync) // If we are limited by the filter AND the history visibility filter // didn't "remove" events, return that the response is limited. jr.Timeline.Limited = limited && len(events) == len(recentEvents) - jr.State.Events = gomatrixserverlib.HeaderedToClientEvents(stateEvents, gomatrixserverlib.FormatSync) + jr.State.Events = synctypes.HeaderedToClientEvents(stateEvents, synctypes.FormatSync) return jr, nil } func (p *PDUStreamProvider) lazyLoadMembers( ctx context.Context, snapshot storage.DatabaseTransaction, roomID string, - incremental, limited bool, stateFilter *gomatrixserverlib.StateFilter, + incremental, limited bool, stateFilter *synctypes.StateFilter, device *userapi.Device, timelineEvents, stateEvents []*gomatrixserverlib.HeaderedEvent, ) ([]*gomatrixserverlib.HeaderedEvent, error) { @@ -595,7 +596,7 @@ func (p *PDUStreamProvider) lazyLoadMembers( wantUsers = append(wantUsers, userID) } // Query missing membership events - filter := gomatrixserverlib.DefaultStateFilter() + filter := synctypes.DefaultStateFilter() filter.Senders = &wantUsers filter.Types = &[]string{gomatrixserverlib.MRoomMember} memberships, err := snapshot.GetStateEventsForRoom(ctx, roomID, &filter) @@ -612,7 +613,7 @@ func (p *PDUStreamProvider) lazyLoadMembers( // addIgnoredUsersToFilter adds ignored users to the eventfilter and // the syncreq itself for further use in streams. -func (p *PDUStreamProvider) addIgnoredUsersToFilter(ctx context.Context, snapshot storage.DatabaseTransaction, req *types.SyncRequest, eventFilter *gomatrixserverlib.RoomEventFilter) error { +func (p *PDUStreamProvider) addIgnoredUsersToFilter(ctx context.Context, snapshot storage.DatabaseTransaction, req *types.SyncRequest, eventFilter *synctypes.RoomEventFilter) error { ignores, err := snapshot.IgnoresForUser(ctx, req.Device.UserID) if err != nil { if err == sql.ErrNoRows { diff --git a/syncapi/streams/stream_presence.go b/syncapi/streams/stream_presence.go index 445e46b3a3..81f33207c7 100644 --- a/syncapi/streams/stream_presence.go +++ b/syncapi/streams/stream_presence.go @@ -25,6 +25,7 @@ import ( "github.com/matrix-org/dendrite/syncapi/notifier" "github.com/matrix-org/dendrite/syncapi/storage" + "github.com/matrix-org/dendrite/syncapi/synctypes" "github.com/matrix-org/dendrite/syncapi/types" ) @@ -65,7 +66,7 @@ func (p *PresenceStreamProvider) IncrementalSync( from, to types.StreamPosition, ) types.StreamPosition { // We pull out a larger number than the filter asks for, since we're filtering out events later - presences, err := snapshot.PresenceAfter(ctx, from, gomatrixserverlib.EventFilter{Limit: 1000}) + presences, err := snapshot.PresenceAfter(ctx, from, synctypes.EventFilter{Limit: 1000}) if err != nil { req.Log.WithError(err).Error("p.DB.PresenceAfter failed") return from @@ -130,7 +131,7 @@ func (p *PresenceStreamProvider) IncrementalSync( return from } - req.Response.Presence.Events = append(req.Response.Presence.Events, gomatrixserverlib.ClientEvent{ + req.Response.Presence.Events = append(req.Response.Presence.Events, synctypes.ClientEvent{ Content: content, Sender: presence.UserID, Type: gomatrixserverlib.MPresence, @@ -202,7 +203,7 @@ func joinedRooms(res *types.Response, userID string) []string { return roomIDs } -func membershipEventPresent(events []gomatrixserverlib.ClientEvent, userID string) bool { +func membershipEventPresent(events []synctypes.ClientEvent, userID string) bool { for _, ev := range events { // it's enough to know that we have our member event here, don't need to check membership content // as it's implied by being in the respective section of the sync response. diff --git a/syncapi/streams/stream_receipt.go b/syncapi/streams/stream_receipt.go index 16a81e833b..88db0054e3 100644 --- a/syncapi/streams/stream_receipt.go +++ b/syncapi/streams/stream_receipt.go @@ -7,6 +7,7 @@ import ( "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/dendrite/syncapi/storage" + "github.com/matrix-org/dendrite/syncapi/synctypes" "github.com/matrix-org/dendrite/syncapi/types" ) @@ -86,7 +87,7 @@ func (p *ReceiptStreamProvider) IncrementalSync( jr = types.NewJoinResponse() } - ev := gomatrixserverlib.ClientEvent{ + ev := synctypes.ClientEvent{ Type: gomatrixserverlib.MReceipt, } content := make(map[string]ReceiptMRead) diff --git a/syncapi/streams/stream_typing.go b/syncapi/streams/stream_typing.go index 84c199b396..b0e7d9e7c2 100644 --- a/syncapi/streams/stream_typing.go +++ b/syncapi/streams/stream_typing.go @@ -8,6 +8,7 @@ import ( "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/syncapi/storage" + "github.com/matrix-org/dendrite/syncapi/synctypes" "github.com/matrix-org/dendrite/syncapi/types" ) @@ -51,7 +52,7 @@ func (p *TypingStreamProvider) IncrementalSync( typingUsers = append(typingUsers, users[i]) } } - ev := gomatrixserverlib.ClientEvent{ + ev := synctypes.ClientEvent{ Type: gomatrixserverlib.MTyping, } ev.Content, err = json.Marshal(map[string]interface{}{ diff --git a/syncapi/sync/request.go b/syncapi/sync/request.go index e5e5fdb5be..6173423315 100644 --- a/syncapi/sync/request.go +++ b/syncapi/sync/request.go @@ -28,6 +28,7 @@ import ( "github.com/sirupsen/logrus" "github.com/matrix-org/dendrite/syncapi/storage" + "github.com/matrix-org/dendrite/syncapi/synctypes" "github.com/matrix-org/dendrite/syncapi/types" userapi "github.com/matrix-org/dendrite/userapi/api" ) @@ -49,7 +50,7 @@ func newSyncRequest(req *http.Request, device userapi.Device, syncDB storage.Dat } // Create a default filter and apply a stored filter on top of it (if specified) - filter := gomatrixserverlib.DefaultFilter() + filter := synctypes.DefaultFilter() filterQuery := req.URL.Query().Get("filter") if filterQuery != "" { if filterQuery[0] == '{' { diff --git a/syncapi/sync/requestpool_test.go b/syncapi/sync/requestpool_test.go index faa0b49c67..7bce0a0c73 100644 --- a/syncapi/sync/requestpool_test.go +++ b/syncapi/sync/requestpool_test.go @@ -7,6 +7,7 @@ import ( "time" "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/dendrite/syncapi/synctypes" "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/gomatrixserverlib" ) @@ -33,7 +34,7 @@ func (d dummyDB) GetPresences(ctx context.Context, userID []string) ([]*types.Pr return []*types.PresenceInternal{}, nil } -func (d dummyDB) PresenceAfter(ctx context.Context, after types.StreamPosition, filter gomatrixserverlib.EventFilter) (map[string]*types.PresenceInternal, error) { +func (d dummyDB) PresenceAfter(ctx context.Context, after types.StreamPosition, filter synctypes.EventFilter) (map[string]*types.PresenceInternal, error) { return map[string]*types.PresenceInternal{}, nil } diff --git a/syncapi/syncapi_test.go b/syncapi/syncapi_test.go index 584782afb2..6496a60706 100644 --- a/syncapi/syncapi_test.go +++ b/syncapi/syncapi_test.go @@ -20,6 +20,7 @@ import ( "github.com/matrix-org/dendrite/syncapi/routing" "github.com/matrix-org/dendrite/syncapi/storage" + "github.com/matrix-org/dendrite/syncapi/synctypes" "github.com/matrix-org/dendrite/clientapi/producers" "github.com/matrix-org/dendrite/roomserver" @@ -473,7 +474,7 @@ func testHistoryVisibility(t *testing.T, dbType test.DBType) { } // We only care about the returned events at this point var res struct { - Chunk []gomatrixserverlib.ClientEvent `json:"chunk"` + Chunk []synctypes.ClientEvent `json:"chunk"` } if err := json.NewDecoder(w.Body).Decode(&res); err != nil { t.Errorf("failed to decode response body: %s", err) @@ -521,7 +522,7 @@ func testHistoryVisibility(t *testing.T, dbType test.DBType) { } } -func verifyEventVisible(t *testing.T, wantVisible bool, wantVisibleEvent *gomatrixserverlib.HeaderedEvent, chunk []gomatrixserverlib.ClientEvent) { +func verifyEventVisible(t *testing.T, wantVisible bool, wantVisibleEvent *gomatrixserverlib.HeaderedEvent, chunk []synctypes.ClientEvent) { t.Helper() if wantVisible { for _, ev := range chunk { diff --git a/syncapi/synctypes/clientevent.go b/syncapi/synctypes/clientevent.go new file mode 100644 index 0000000000..0d1e85bcc5 --- /dev/null +++ b/syncapi/synctypes/clientevent.go @@ -0,0 +1,88 @@ +/* Copyright 2017 Vector Creations Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package synctypes + +import "github.com/matrix-org/gomatrixserverlib" + +type ClientEventFormat int + +const ( + // FormatAll will include all client event keys + FormatAll ClientEventFormat = iota + // FormatSync will include only the event keys required by the /sync API. Notably, this + // means the 'room_id' will be missing from the events. + FormatSync +) + +// ClientEvent is an event which is fit for consumption by clients, in accordance with the specification. +type ClientEvent struct { + Content gomatrixserverlib.RawJSON `json:"content"` + EventID string `json:"event_id,omitempty"` // EventID is omitted on receipt events + OriginServerTS gomatrixserverlib.Timestamp `json:"origin_server_ts,omitempty"` // OriginServerTS is omitted on receipt events + RoomID string `json:"room_id,omitempty"` // RoomID is omitted on /sync responses + Sender string `json:"sender,omitempty"` // Sender is omitted on receipt events + StateKey *string `json:"state_key,omitempty"` + Type string `json:"type"` + Unsigned gomatrixserverlib.RawJSON `json:"unsigned,omitempty"` + Redacts string `json:"redacts,omitempty"` +} + +// ToClientEvents converts server events to client events. +func ToClientEvents(serverEvs []*gomatrixserverlib.Event, format ClientEventFormat) []ClientEvent { + evs := make([]ClientEvent, 0, len(serverEvs)) + for _, se := range serverEvs { + if se == nil { + continue // TODO: shouldn't happen? + } + evs = append(evs, ToClientEvent(se, format)) + } + return evs +} + +// HeaderedToClientEvents converts headered server events to client events. +func HeaderedToClientEvents(serverEvs []*gomatrixserverlib.HeaderedEvent, format ClientEventFormat) []ClientEvent { + evs := make([]ClientEvent, 0, len(serverEvs)) + for _, se := range serverEvs { + if se == nil { + continue // TODO: shouldn't happen? + } + evs = append(evs, HeaderedToClientEvent(se, format)) + } + return evs +} + +// ToClientEvent converts a single server event to a client event. +func ToClientEvent(se *gomatrixserverlib.Event, format ClientEventFormat) ClientEvent { + ce := ClientEvent{ + Content: gomatrixserverlib.RawJSON(se.Content()), + Sender: se.Sender(), + Type: se.Type(), + StateKey: se.StateKey(), + Unsigned: gomatrixserverlib.RawJSON(se.Unsigned()), + OriginServerTS: se.OriginServerTS(), + EventID: se.EventID(), + Redacts: se.Redacts(), + } + if format == FormatAll { + ce.RoomID = se.RoomID() + } + return ce +} + +// HeaderedToClientEvent converts a single headered server event to a client event. +func HeaderedToClientEvent(se *gomatrixserverlib.HeaderedEvent, format ClientEventFormat) ClientEvent { + return ToClientEvent(se.Event, format) +} diff --git a/syncapi/synctypes/clientevent_test.go b/syncapi/synctypes/clientevent_test.go new file mode 100644 index 0000000000..ac07917ab0 --- /dev/null +++ b/syncapi/synctypes/clientevent_test.go @@ -0,0 +1,105 @@ +/* Copyright 2017 Vector Creations Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package synctypes + +import ( + "bytes" + "encoding/json" + "testing" + + "github.com/matrix-org/gomatrixserverlib" +) + +func TestToClientEvent(t *testing.T) { // nolint: gocyclo + ev, err := gomatrixserverlib.NewEventFromTrustedJSON([]byte(`{ + "type": "m.room.name", + "state_key": "", + "event_id": "$test:localhost", + "room_id": "!test:localhost", + "sender": "@test:localhost", + "content": { + "name": "Hello World" + }, + "origin_server_ts": 123456, + "unsigned": { + "prev_content": { + "name": "Goodbye World" + } + } + }`), false, gomatrixserverlib.RoomVersionV1) + if err != nil { + t.Fatalf("failed to create Event: %s", err) + } + ce := ToClientEvent(ev, FormatAll) + if ce.EventID != ev.EventID() { + t.Errorf("ClientEvent.EventID: wanted %s, got %s", ev.EventID(), ce.EventID) + } + if ce.OriginServerTS != ev.OriginServerTS() { + t.Errorf("ClientEvent.OriginServerTS: wanted %d, got %d", ev.OriginServerTS(), ce.OriginServerTS) + } + if ce.StateKey == nil || *ce.StateKey != "" { + t.Errorf("ClientEvent.StateKey: wanted '', got %v", ce.StateKey) + } + if ce.Type != ev.Type() { + t.Errorf("ClientEvent.Type: wanted %s, got %s", ev.Type(), ce.Type) + } + if !bytes.Equal(ce.Content, ev.Content()) { + t.Errorf("ClientEvent.Content: wanted %s, got %s", string(ev.Content()), string(ce.Content)) + } + if !bytes.Equal(ce.Unsigned, ev.Unsigned()) { + t.Errorf("ClientEvent.Unsigned: wanted %s, got %s", string(ev.Unsigned()), string(ce.Unsigned)) + } + if ce.Sender != ev.Sender() { + t.Errorf("ClientEvent.Sender: wanted %s, got %s", ev.Sender(), ce.Sender) + } + j, err := json.Marshal(ce) + if err != nil { + t.Fatalf("failed to Marshal ClientEvent: %s", err) + } + // Marshal sorts keys in structs by the order they are defined in the struct, which is alphabetical + out := `{"content":{"name":"Hello World"},"event_id":"$test:localhost","origin_server_ts":123456,` + + `"room_id":"!test:localhost","sender":"@test:localhost","state_key":"","type":"m.room.name",` + + `"unsigned":{"prev_content":{"name":"Goodbye World"}}}` + if !bytes.Equal([]byte(out), j) { + t.Errorf("ClientEvent marshalled to wrong bytes: wanted %s, got %s", out, string(j)) + } +} + +func TestToClientFormatSync(t *testing.T) { + ev, err := gomatrixserverlib.NewEventFromTrustedJSON([]byte(`{ + "type": "m.room.name", + "state_key": "", + "event_id": "$test:localhost", + "room_id": "!test:localhost", + "sender": "@test:localhost", + "content": { + "name": "Hello World" + }, + "origin_server_ts": 123456, + "unsigned": { + "prev_content": { + "name": "Goodbye World" + } + } + }`), false, gomatrixserverlib.RoomVersionV1) + if err != nil { + t.Fatalf("failed to create Event: %s", err) + } + ce := ToClientEvent(ev, FormatSync) + if ce.RoomID != "" { + t.Errorf("ClientEvent.RoomID: wanted '', got %s", ce.RoomID) + } +} diff --git a/syncapi/synctypes/filter.go b/syncapi/synctypes/filter.go new file mode 100644 index 0000000000..c994ddb96f --- /dev/null +++ b/syncapi/synctypes/filter.go @@ -0,0 +1,152 @@ +// Copyright 2017 Jan Christian Grünhage +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package synctypes + +import ( + "errors" +) + +// Filter is used by clients to specify how the server should filter responses to e.g. sync requests +// Specified by: https://spec.matrix.org/v1.6/client-server-api/#filtering +type Filter struct { + EventFields []string `json:"event_fields,omitempty"` + EventFormat string `json:"event_format,omitempty"` + Presence EventFilter `json:"presence,omitempty"` + AccountData EventFilter `json:"account_data,omitempty"` + Room RoomFilter `json:"room,omitempty"` +} + +// EventFilter is used to define filtering rules for events +type EventFilter struct { + Limit int `json:"limit,omitempty"` + NotSenders *[]string `json:"not_senders,omitempty"` + NotTypes *[]string `json:"not_types,omitempty"` + Senders *[]string `json:"senders,omitempty"` + Types *[]string `json:"types,omitempty"` +} + +// RoomFilter is used to define filtering rules for room-related events +type RoomFilter struct { + NotRooms *[]string `json:"not_rooms,omitempty"` + Rooms *[]string `json:"rooms,omitempty"` + Ephemeral RoomEventFilter `json:"ephemeral,omitempty"` + IncludeLeave bool `json:"include_leave,omitempty"` + State StateFilter `json:"state,omitempty"` + Timeline RoomEventFilter `json:"timeline,omitempty"` + AccountData RoomEventFilter `json:"account_data,omitempty"` +} + +// StateFilter is used to define filtering rules for state events +type StateFilter struct { + NotSenders *[]string `json:"not_senders,omitempty"` + NotTypes *[]string `json:"not_types,omitempty"` + Senders *[]string `json:"senders,omitempty"` + Types *[]string `json:"types,omitempty"` + LazyLoadMembers bool `json:"lazy_load_members,omitempty"` + IncludeRedundantMembers bool `json:"include_redundant_members,omitempty"` + NotRooms *[]string `json:"not_rooms,omitempty"` + Rooms *[]string `json:"rooms,omitempty"` + Limit int `json:"limit,omitempty"` + UnreadThreadNotifications bool `json:"unread_thread_notifications,omitempty"` + ContainsURL *bool `json:"contains_url,omitempty"` +} + +// RoomEventFilter is used to define filtering rules for events in rooms +type RoomEventFilter struct { + Limit int `json:"limit,omitempty"` + NotSenders *[]string `json:"not_senders,omitempty"` + NotTypes *[]string `json:"not_types,omitempty"` + Senders *[]string `json:"senders,omitempty"` + Types *[]string `json:"types,omitempty"` + LazyLoadMembers bool `json:"lazy_load_members,omitempty"` + IncludeRedundantMembers bool `json:"include_redundant_members,omitempty"` + NotRooms *[]string `json:"not_rooms,omitempty"` + Rooms *[]string `json:"rooms,omitempty"` + UnreadThreadNotifications bool `json:"unread_thread_notifications,omitempty"` + ContainsURL *bool `json:"contains_url,omitempty"` +} + +// Validate checks if the filter contains valid property values +func (filter *Filter) Validate() error { + if filter.EventFormat != "" && filter.EventFormat != "client" && filter.EventFormat != "federation" { + return errors.New("Bad event_format value. Must be one of [\"client\", \"federation\"]") + } + return nil +} + +// DefaultFilter returns the default filter used by the Matrix server if no filter is provided in +// the request +func DefaultFilter() Filter { + return Filter{ + AccountData: DefaultEventFilter(), + EventFields: nil, + EventFormat: "client", + Presence: DefaultEventFilter(), + Room: RoomFilter{ + AccountData: DefaultRoomEventFilter(), + Ephemeral: DefaultRoomEventFilter(), + IncludeLeave: false, + NotRooms: nil, + Rooms: nil, + State: DefaultStateFilter(), + Timeline: DefaultRoomEventFilter(), + }, + } +} + +// DefaultEventFilter returns the default event filter used by the Matrix server if no filter is +// provided in the request +func DefaultEventFilter() EventFilter { + return EventFilter{ + // parity with synapse: https://github.com/matrix-org/synapse/blob/v1.80.0/synapse/api/filtering.py#L336 + Limit: 10, + NotSenders: nil, + NotTypes: nil, + Senders: nil, + Types: nil, + } +} + +// DefaultStateFilter returns the default state event filter used by the Matrix server if no filter +// is provided in the request +func DefaultStateFilter() StateFilter { + return StateFilter{ + NotSenders: nil, + NotTypes: nil, + Senders: nil, + Types: nil, + LazyLoadMembers: false, + IncludeRedundantMembers: false, + NotRooms: nil, + Rooms: nil, + ContainsURL: nil, + } +} + +// DefaultRoomEventFilter returns the default room event filter used by the Matrix server if no +// filter is provided in the request +func DefaultRoomEventFilter() RoomEventFilter { + return RoomEventFilter{ + // parity with synapse: https://github.com/matrix-org/synapse/blob/v1.80.0/synapse/api/filtering.py#L336 + Limit: 10, + NotSenders: nil, + NotTypes: nil, + Senders: nil, + Types: nil, + NotRooms: nil, + Rooms: nil, + ContainsURL: nil, + } +} diff --git a/syncapi/synctypes/filter_test.go b/syncapi/synctypes/filter_test.go new file mode 100644 index 0000000000..d00ee69d6b --- /dev/null +++ b/syncapi/synctypes/filter_test.go @@ -0,0 +1,60 @@ +package synctypes + +import ( + "encoding/json" + "reflect" + "testing" +) + +func Test_Filter(t *testing.T) { + tests := []struct { + name string + input []byte + want RoomEventFilter + }{ + { + name: "empty types filter", + input: []byte(`{ "types": [] }`), + want: RoomEventFilter{ + Limit: 0, + NotSenders: nil, + NotTypes: nil, + Senders: nil, + Types: &[]string{}, + LazyLoadMembers: false, + IncludeRedundantMembers: false, + NotRooms: nil, + Rooms: nil, + ContainsURL: nil, + }, + }, + { + name: "absent types filter", + input: []byte(`{}`), + want: RoomEventFilter{ + Limit: 0, + NotSenders: nil, + NotTypes: nil, + Senders: nil, + Types: nil, + LazyLoadMembers: false, + IncludeRedundantMembers: false, + NotRooms: nil, + Rooms: nil, + ContainsURL: nil, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var f RoomEventFilter + if err := json.Unmarshal(tt.input, &f); err != nil { + t.Fatalf("unable to parse filter: %v", err) + } + if !reflect.DeepEqual(f, tt.want) { + t.Fatalf("Expected %+v\ngot %+v", tt.want, f) + } + }) + } + +} diff --git a/syncapi/types/provider.go b/syncapi/types/provider.go index 9a533002b9..f77f83422a 100644 --- a/syncapi/types/provider.go +++ b/syncapi/types/provider.go @@ -7,6 +7,7 @@ import ( "github.com/matrix-org/gomatrixserverlib" "github.com/sirupsen/logrus" + "github.com/matrix-org/dendrite/syncapi/synctypes" userapi "github.com/matrix-org/dendrite/userapi/api" ) @@ -15,7 +16,7 @@ type SyncRequest struct { Log *logrus.Entry Device *userapi.Device Response *Response - Filter gomatrixserverlib.Filter + Filter synctypes.Filter Since StreamingToken Timeout time.Duration WantFullState bool diff --git a/syncapi/types/types.go b/syncapi/types/types.go index 6495dd535d..0c522fc3b7 100644 --- a/syncapi/types/types.go +++ b/syncapi/types/types.go @@ -25,6 +25,7 @@ import ( "github.com/tidwall/gjson" "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/syncapi/synctypes" ) var ( @@ -451,13 +452,13 @@ type UnreadNotifications struct { } type ClientEvents struct { - Events []gomatrixserverlib.ClientEvent `json:"events,omitempty"` + Events []synctypes.ClientEvent `json:"events,omitempty"` } type Timeline struct { - Events []gomatrixserverlib.ClientEvent `json:"events"` - Limited bool `json:"limited"` - PrevBatch *TopologyToken `json:"prev_batch,omitempty"` + Events []synctypes.ClientEvent `json:"events"` + Limited bool `json:"limited"` + PrevBatch *TopologyToken `json:"prev_batch,omitempty"` } type Summary struct { @@ -549,7 +550,7 @@ func NewInviteResponse(event *gomatrixserverlib.HeaderedEvent) *InviteResponse { // Then we'll see if we can create a partial of the invite event itself. // This is needed for clients to work out *who* sent the invite. - inviteEvent := gomatrixserverlib.ToClientEvent(event.Unwrap(), gomatrixserverlib.FormatSync) + inviteEvent := synctypes.ToClientEvent(event.Unwrap(), synctypes.FormatSync) inviteEvent.Unsigned = nil if ev, err := json.Marshal(inviteEvent); err == nil { res.InviteState.Events = append(res.InviteState.Events, ev) diff --git a/syncapi/types/types_test.go b/syncapi/types/types_test.go index 74246d9642..8bb887ffe0 100644 --- a/syncapi/types/types_test.go +++ b/syncapi/types/types_test.go @@ -5,6 +5,7 @@ import ( "reflect" "testing" + "github.com/matrix-org/dendrite/syncapi/synctypes" "github.com/matrix-org/gomatrixserverlib" ) @@ -125,7 +126,7 @@ func TestJoinResponse_MarshalJSON(t *testing.T) { { name: "unread notifications are NOT removed, if state is set", fields: fields{ - State: &ClientEvents{Events: []gomatrixserverlib.ClientEvent{{Content: []byte("{}")}}}, + State: &ClientEvents{Events: []synctypes.ClientEvent{{Content: []byte("{}")}}}, UnreadNotifications: &UnreadNotifications{NotificationCount: 1}, }, want: []byte(`{"state":{"events":[{"content":{},"type":""}]},"unread_notifications":{"highlight_count":0,"notification_count":1}}`), @@ -134,7 +135,7 @@ func TestJoinResponse_MarshalJSON(t *testing.T) { name: "roomID is removed from EDUs", fields: fields{ Ephemeral: &ClientEvents{ - Events: []gomatrixserverlib.ClientEvent{ + Events: []synctypes.ClientEvent{ {RoomID: "!someRandomRoomID:test", Content: []byte("{}")}, }, }, diff --git a/userapi/api/api.go b/userapi/api/api.go index 5fd7992c8a..f65f81ad3b 100644 --- a/userapi/api/api.go +++ b/userapi/api/api.go @@ -21,6 +21,7 @@ import ( "strings" "time" + "github.com/matrix-org/dendrite/syncapi/synctypes" "github.com/matrix-org/dendrite/userapi/types" "github.com/matrix-org/gomatrixserverlib" @@ -580,12 +581,12 @@ type QueryNotificationsResponse struct { } type Notification struct { - Actions []*pushrules.Action `json:"actions"` // Required. - Event gomatrixserverlib.ClientEvent `json:"event"` // Required. - ProfileTag string `json:"profile_tag"` // Required by Sytest, but actually optional. - Read bool `json:"read"` // Required. - RoomID string `json:"room_id"` // Required. - TS gomatrixserverlib.Timestamp `json:"ts"` // Required. + Actions []*pushrules.Action `json:"actions"` // Required. + Event synctypes.ClientEvent `json:"event"` // Required. + ProfileTag string `json:"profile_tag"` // Required by Sytest, but actually optional. + Read bool `json:"read"` // Required. + RoomID string `json:"room_id"` // Required. + TS gomatrixserverlib.Timestamp `json:"ts"` // Required. } type QueryNumericLocalpartRequest struct { diff --git a/userapi/consumers/roomserver.go b/userapi/consumers/roomserver.go index 47d330959d..6704658df1 100644 --- a/userapi/consumers/roomserver.go +++ b/userapi/consumers/roomserver.go @@ -23,6 +23,7 @@ import ( "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/jetstream" "github.com/matrix-org/dendrite/setup/process" + "github.com/matrix-org/dendrite/syncapi/synctypes" "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/dendrite/userapi/producers" @@ -298,7 +299,7 @@ func (s *OutputRoomEventConsumer) processMessage(ctx context.Context, event *gom switch { case event.Type() == gomatrixserverlib.MRoomMember: - cevent := gomatrixserverlib.HeaderedToClientEvent(event, gomatrixserverlib.FormatAll) + cevent := synctypes.HeaderedToClientEvent(event, synctypes.FormatAll) var member *localMembership member, err = newLocalMembership(&cevent) if err != nil { @@ -358,7 +359,7 @@ type localMembership struct { Domain gomatrixserverlib.ServerName } -func newLocalMembership(event *gomatrixserverlib.ClientEvent) (*localMembership, error) { +func newLocalMembership(event *synctypes.ClientEvent) (*localMembership, error) { if event.StateKey == nil { return nil, fmt.Errorf("missing state_key") } @@ -531,7 +532,7 @@ func (s *OutputRoomEventConsumer) notifyLocal(ctx context.Context, event *gomatr // UNSPEC: the spec doesn't say this is a ClientEvent, but the // fields seem to match. room_id should be missing, which // matches the behaviour of FormatSync. - Event: gomatrixserverlib.HeaderedToClientEvent(event, gomatrixserverlib.FormatSync), + Event: synctypes.HeaderedToClientEvent(event, synctypes.FormatSync), // TODO: this is per-device, but it's not part of the primary // key. So inserting one notification per profile tag doesn't // make sense. What is this supposed to be? Sytests require it diff --git a/userapi/storage/storage_test.go b/userapi/storage/storage_test.go index cf7c5144eb..7afcda6e9e 100644 --- a/userapi/storage/storage_test.go +++ b/userapi/storage/storage_test.go @@ -10,6 +10,7 @@ import ( "time" "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/syncapi/synctypes" "github.com/matrix-org/dendrite/userapi/types" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" @@ -356,12 +357,12 @@ func Test_OpenID(t *testing.T) { expiresAtMS := time.Now().UnixNano()/int64(time.Millisecond) + openIDLifetimeMS expires, err := db.CreateOpenIDToken(ctx, token, alice.ID) assert.NoError(t, err, "unable to create OpenID token") - assert.Equal(t, expiresAtMS, expires) + assert.InDelta(t, expiresAtMS, expires, 2) // 2ms leeway attributes, err := db.GetOpenIDTokenAttributes(ctx, token) assert.NoError(t, err, "unable to get OpenID token attributes") assert.Equal(t, alice.ID, attributes.UserID) - assert.Equal(t, expiresAtMS, attributes.ExpiresAtMS) + assert.InDelta(t, expiresAtMS, attributes.ExpiresAtMS, 2) // 2ms leeway }) } @@ -526,7 +527,7 @@ func Test_Notification(t *testing.T) { Actions: []*pushrules.Action{ {}, }, - Event: gomatrixserverlib.ClientEvent{ + Event: synctypes.ClientEvent{ Content: gomatrixserverlib.RawJSON("{}"), }, Read: false, diff --git a/userapi/util/notify_test.go b/userapi/util/notify_test.go index 69461ddd1e..d6cbad7db8 100644 --- a/userapi/util/notify_test.go +++ b/userapi/util/notify_test.go @@ -9,6 +9,7 @@ import ( "time" "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/syncapi/synctypes" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" "golang.org/x/crypto/bcrypt" @@ -99,7 +100,7 @@ func TestNotifyUserCountsAsync(t *testing.T) { // Insert a dummy event if err := db.InsertNotification(ctx, aliceLocalpart, serverName, dummyEvent.EventID(), 0, nil, &api.Notification{ - Event: gomatrixserverlib.HeaderedToClientEvent(dummyEvent, gomatrixserverlib.FormatAll), + Event: synctypes.HeaderedToClientEvent(dummyEvent, synctypes.FormatAll), }); err != nil { t.Error(err) } From e093005bc2a2a582ed884826fc4efc90c4b9d9ad Mon Sep 17 00:00:00 2001 From: kegsay Date: Wed, 5 Apr 2023 14:35:55 +0100 Subject: [PATCH 33/42] ci: don't use go get, use go install (#3048) Otherwise CI can fail with: ``` go: go.mod file not found in current directory or any parent directory. 'go get' is no longer supported outside a module. To build and install a command, use 'go install' with a version, like 'go install example.com/cmd@latest' For more information, see https://golang.org/doc/go-get-install-deprecation or run 'go help get' or 'go help install'. ``` --- .github/workflows/dendrite.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dendrite.yml b/.github/workflows/dendrite.yml index 2101c7a509..17f02b09e3 100644 --- a/.github/workflows/dendrite.yml +++ b/.github/workflows/dendrite.yml @@ -393,7 +393,7 @@ jobs: # See https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2004-Readme.md specifically GOROOT_1_17_X64 run: | sudo apt-get update && sudo apt-get install -y libolm3 libolm-dev - go get -v github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt@latest + go install github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt@latest - name: Run actions/checkout@v3 for dendrite uses: actions/checkout@v3 with: From 0db43f13a6b79cc2bd3e210051800e4d6de87c42 Mon Sep 17 00:00:00 2001 From: kegsay Date: Thu, 6 Apr 2023 09:55:01 +0100 Subject: [PATCH 34/42] refactor: use latest GMSL which splits fed client from matrix room logic (#3051) Part of a series of refactors on GMSL. --- clientapi/admin_test.go | 3 +- clientapi/api/api.go | 4 +- clientapi/auth/login_test.go | 6 +- clientapi/auth/user_interactive_test.go | 3 +- clientapi/clientapi.go | 4 +- clientapi/routing/directory.go | 3 +- clientapi/routing/directory_public.go | 31 +++---- clientapi/routing/directory_public_test.go | 10 +-- clientapi/routing/login_test.go | 3 +- clientapi/routing/profile.go | 9 ++- clientapi/routing/routing.go | 3 +- clientapi/routing/threepid.go | 5 +- clientapi/routing/userdirectory.go | 3 +- clientapi/threepid/threepid.go | 8 +- clientapi/userutil/userutil_test.go | 9 ++- cmd/dendrite-demo-pinecone/conn/client.go | 14 ++-- cmd/dendrite-demo-pinecone/rooms/rooms.go | 13 +-- cmd/dendrite-demo-pinecone/users/users.go | 7 +- cmd/dendrite-demo-yggdrasil/yggconn/client.go | 14 ++-- .../yggrooms/yggrooms.go | 13 +-- cmd/dendrite/main.go | 6 +- cmd/furl/main.go | 5 +- federationapi/api/api.go | 51 ++++++------ federationapi/federationapi.go | 3 +- federationapi/federationapi_keys_test.go | 17 ++-- federationapi/federationapi_test.go | 11 +-- federationapi/internal/federationclient.go | 56 ++++++------- .../internal/federationclient_test.go | 23 +++--- federationapi/internal/perform.go | 3 +- federationapi/internal/perform_test.go | 15 ++-- federationapi/queue/destinationqueue.go | 3 +- federationapi/queue/queue.go | 7 +- federationapi/queue/queue_test.go | 11 +-- federationapi/routing/devices.go | 9 ++- federationapi/routing/eventauth.go | 3 +- federationapi/routing/invite.go | 5 +- federationapi/routing/join.go | 3 +- federationapi/routing/keys.go | 3 +- federationapi/routing/missingevents.go | 3 +- federationapi/routing/peek.go | 3 +- federationapi/routing/publicrooms.go | 11 +-- federationapi/routing/query.go | 5 +- federationapi/routing/query_test.go | 3 +- federationapi/routing/state.go | 5 +- go.mod | 2 +- go.sum | 2 + internal/caching/cache_space_rooms.go | 12 ++- internal/caching/caches.go | 3 +- internal/caching/impl_ristretto.go | 3 +- internal/eventutil/events.go | 5 +- internal/transactionrequest.go | 15 ++-- mediaapi/mediaapi.go | 4 +- mediaapi/routing/download.go | 11 +-- mediaapi/routing/routing.go | 5 +- relayapi/api/api.go | 3 +- relayapi/internal/perform.go | 9 ++- relayapi/internal/perform_test.go | 7 +- relayapi/relayapi.go | 3 +- relayapi/relayapi_test.go | 3 +- relayapi/routing/relaytxn.go | 5 +- relayapi/routing/relaytxn_test.go | 29 +++---- relayapi/routing/sendrelay.go | 3 +- roomserver/api/wrapper.go | 9 ++- roomserver/internal/input/input.go | 3 +- roomserver/internal/input/input_events.go | 3 +- roomserver/internal/input/input_missing.go | 45 ++++++----- roomserver/internal/perform/perform_admin.go | 6 +- setup/base/base.go | 36 ++++----- setup/config/config_global.go | 11 +-- setup/config/config_test.go | 13 +-- setup/monolith.go | 5 +- setup/mscs/msc2836/msc2836.go | 9 ++- setup/mscs/msc2946/msc2946.go | 31 +++---- userapi/api/api.go | 23 +++--- userapi/consumers/signingkeyupdate.go | 3 +- userapi/internal/cross_signing.go | 81 ++++++++++--------- userapi/internal/device_list_update.go | 7 +- userapi/internal/device_list_update_test.go | 11 +-- userapi/internal/key_api.go | 15 ++-- userapi/storage/interface.go | 3 +- .../postgres/cross_signing_keys_table.go | 3 +- userapi/storage/shared/storage.go | 9 ++- .../sqlite3/cross_signing_keys_table.go | 3 +- userapi/storage/tables/interface.go | 3 +- userapi/types/storage.go | 19 ++--- userapi/userapi_test.go | 3 +- 86 files changed, 493 insertions(+), 414 deletions(-) diff --git a/clientapi/admin_test.go b/clientapi/admin_test.go index ca78f32e3d..da1ac70b91 100644 --- a/clientapi/admin_test.go +++ b/clientapi/admin_test.go @@ -19,6 +19,7 @@ import ( "github.com/matrix-org/dendrite/setup/jetstream" "github.com/matrix-org/dendrite/syncapi" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/matrix-org/util" "github.com/tidwall/gjson" @@ -40,7 +41,7 @@ func TestAdminResetPassword(t *testing.T) { natsInstance := jetstream.NATSInstance{} // add a vhost cfg.Global.VirtualHosts = append(cfg.Global.VirtualHosts, &config.VirtualHost{ - SigningIdentity: gomatrixserverlib.SigningIdentity{ServerName: "vh1"}, + SigningIdentity: fclient.SigningIdentity{ServerName: "vh1"}, }) routers := httputil.NewRouters() diff --git a/clientapi/api/api.go b/clientapi/api/api.go index d96b032f01..23974c8658 100644 --- a/clientapi/api/api.go +++ b/clientapi/api/api.go @@ -14,10 +14,10 @@ package api -import "github.com/matrix-org/gomatrixserverlib" +import "github.com/matrix-org/gomatrixserverlib/fclient" // ExtraPublicRoomsProvider provides a way to inject extra published rooms into /publicRooms requests. type ExtraPublicRoomsProvider interface { // Rooms returns the extra rooms. This is called on-demand by clients, so cache appropriately. - Rooms() []gomatrixserverlib.PublicRoom + Rooms() []fclient.PublicRoom } diff --git a/clientapi/auth/login_test.go b/clientapi/auth/login_test.go index 044062c420..c91cba2413 100644 --- a/clientapi/auth/login_test.go +++ b/clientapi/auth/login_test.go @@ -25,7 +25,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/userutil" "github.com/matrix-org/dendrite/setup/config" uapi "github.com/matrix-org/dendrite/userapi/api" - "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/matrix-org/util" ) @@ -68,7 +68,7 @@ func TestLoginFromJSONReader(t *testing.T) { var userAPI fakeUserInternalAPI cfg := &config.ClientAPI{ Matrix: &config.Global{ - SigningIdentity: gomatrixserverlib.SigningIdentity{ + SigningIdentity: fclient.SigningIdentity{ ServerName: serverName, }, }, @@ -148,7 +148,7 @@ func TestBadLoginFromJSONReader(t *testing.T) { var userAPI fakeUserInternalAPI cfg := &config.ClientAPI{ Matrix: &config.Global{ - SigningIdentity: gomatrixserverlib.SigningIdentity{ + SigningIdentity: fclient.SigningIdentity{ ServerName: serverName, }, }, diff --git a/clientapi/auth/user_interactive_test.go b/clientapi/auth/user_interactive_test.go index 5d97b31ce7..383a533996 100644 --- a/clientapi/auth/user_interactive_test.go +++ b/clientapi/auth/user_interactive_test.go @@ -9,6 +9,7 @@ import ( "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/matrix-org/util" ) @@ -47,7 +48,7 @@ func (d *fakeAccountDatabase) QueryAccountByPassword(ctx context.Context, req *a func setup() *UserInteractive { cfg := &config.ClientAPI{ Matrix: &config.Global{ - SigningIdentity: gomatrixserverlib.SigningIdentity{ + SigningIdentity: fclient.SigningIdentity{ ServerName: serverName, }, }, diff --git a/clientapi/clientapi.go b/clientapi/clientapi.go index d35f1d4d14..b57c8061e5 100644 --- a/clientapi/clientapi.go +++ b/clientapi/clientapi.go @@ -19,7 +19,7 @@ import ( "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/process" userapi "github.com/matrix-org/dendrite/userapi/api" - "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/fclient" appserviceAPI "github.com/matrix-org/dendrite/appservice/api" "github.com/matrix-org/dendrite/clientapi/api" @@ -37,7 +37,7 @@ func AddPublicRoutes( routers httputil.Routers, cfg *config.Dendrite, natsInstance *jetstream.NATSInstance, - federation *gomatrixserverlib.FederationClient, + federation *fclient.FederationClient, rsAPI roomserverAPI.ClientRoomserverAPI, asAPI appserviceAPI.AppServiceInternalAPI, transactionsCache *transactions.Cache, diff --git a/clientapi/routing/directory.go b/clientapi/routing/directory.go index b3c5aae451..696f0c1ef6 100644 --- a/clientapi/routing/directory.go +++ b/clientapi/routing/directory.go @@ -19,6 +19,7 @@ import ( "net/http" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/matrix-org/util" "github.com/matrix-org/dendrite/clientapi/httputil" @@ -45,7 +46,7 @@ func (r *roomDirectoryResponse) fillServers(servers []gomatrixserverlib.ServerNa func DirectoryRoom( req *http.Request, roomAlias string, - federation *gomatrixserverlib.FederationClient, + federation *fclient.FederationClient, cfg *config.ClientAPI, rsAPI roomserverAPI.ClientRoomserverAPI, fedSenderAPI federationAPI.ClientFederationAPI, diff --git a/clientapi/routing/directory_public.go b/clientapi/routing/directory_public.go index 6067447675..8e1e05a532 100644 --- a/clientapi/routing/directory_public.go +++ b/clientapi/routing/directory_public.go @@ -24,6 +24,7 @@ import ( "sync" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/matrix-org/util" "github.com/matrix-org/dendrite/clientapi/api" @@ -35,7 +36,7 @@ import ( var ( cacheMu sync.Mutex - publicRoomsCache []gomatrixserverlib.PublicRoom + publicRoomsCache []fclient.PublicRoom ) type PublicRoomReq struct { @@ -56,7 +57,7 @@ type filter struct { func GetPostPublicRooms( req *http.Request, rsAPI roomserverAPI.ClientRoomserverAPI, extRoomsProvider api.ExtraPublicRoomsProvider, - federation *gomatrixserverlib.FederationClient, + federation *fclient.FederationClient, cfg *config.ClientAPI, ) util.JSONResponse { var request PublicRoomReq @@ -102,10 +103,10 @@ func GetPostPublicRooms( func publicRooms( ctx context.Context, request PublicRoomReq, rsAPI roomserverAPI.ClientRoomserverAPI, extRoomsProvider api.ExtraPublicRoomsProvider, -) (*gomatrixserverlib.RespPublicRooms, error) { +) (*fclient.RespPublicRooms, error) { - response := gomatrixserverlib.RespPublicRooms{ - Chunk: []gomatrixserverlib.PublicRoom{}, + response := fclient.RespPublicRooms{ + Chunk: []fclient.PublicRoom{}, } var limit int64 var offset int64 @@ -122,7 +123,7 @@ func publicRooms( } err = nil - var rooms []gomatrixserverlib.PublicRoom + var rooms []fclient.PublicRoom if request.Since == "" { rooms = refreshPublicRoomCache(ctx, rsAPI, extRoomsProvider, request) } else { @@ -146,14 +147,14 @@ func publicRooms( return &response, err } -func filterRooms(rooms []gomatrixserverlib.PublicRoom, searchTerm string) []gomatrixserverlib.PublicRoom { +func filterRooms(rooms []fclient.PublicRoom, searchTerm string) []fclient.PublicRoom { if searchTerm == "" { return rooms } normalizedTerm := strings.ToLower(searchTerm) - result := make([]gomatrixserverlib.PublicRoom, 0) + result := make([]fclient.PublicRoom, 0) for _, room := range rooms { if strings.Contains(strings.ToLower(room.Name), normalizedTerm) || strings.Contains(strings.ToLower(room.Topic), normalizedTerm) || @@ -214,7 +215,7 @@ func fillPublicRoomsReq(httpReq *http.Request, request *PublicRoomReq) *util.JSO // limit=3&since=6 => G (prev='3', next='') // // A value of '-1' for prev/next indicates no position. -func sliceInto(slice []gomatrixserverlib.PublicRoom, since int64, limit int64) (subset []gomatrixserverlib.PublicRoom, prev, next int) { +func sliceInto(slice []fclient.PublicRoom, since int64, limit int64) (subset []fclient.PublicRoom, prev, next int) { prev = -1 next = -1 @@ -241,10 +242,10 @@ func sliceInto(slice []gomatrixserverlib.PublicRoom, since int64, limit int64) ( func refreshPublicRoomCache( ctx context.Context, rsAPI roomserverAPI.ClientRoomserverAPI, extRoomsProvider api.ExtraPublicRoomsProvider, request PublicRoomReq, -) []gomatrixserverlib.PublicRoom { +) []fclient.PublicRoom { cacheMu.Lock() defer cacheMu.Unlock() - var extraRooms []gomatrixserverlib.PublicRoom + var extraRooms []fclient.PublicRoom if extRoomsProvider != nil { extraRooms = extRoomsProvider.Rooms() } @@ -269,7 +270,7 @@ func refreshPublicRoomCache( util.GetLogger(ctx).WithError(err).Error("PopulatePublicRooms failed") return publicRoomsCache } - publicRoomsCache = []gomatrixserverlib.PublicRoom{} + publicRoomsCache = []fclient.PublicRoom{} publicRoomsCache = append(publicRoomsCache, pubRooms...) publicRoomsCache = append(publicRoomsCache, extraRooms...) publicRoomsCache = dedupeAndShuffle(publicRoomsCache) @@ -281,16 +282,16 @@ func refreshPublicRoomCache( return publicRoomsCache } -func getPublicRoomsFromCache() []gomatrixserverlib.PublicRoom { +func getPublicRoomsFromCache() []fclient.PublicRoom { cacheMu.Lock() defer cacheMu.Unlock() return publicRoomsCache } -func dedupeAndShuffle(in []gomatrixserverlib.PublicRoom) []gomatrixserverlib.PublicRoom { +func dedupeAndShuffle(in []fclient.PublicRoom) []fclient.PublicRoom { // de-duplicate rooms with the same room ID. We can join the room via any of these aliases as we know these servers // are alive and well, so we arbitrarily pick one (purposefully shuffling them to spread the load a bit) - var publicRooms []gomatrixserverlib.PublicRoom + var publicRooms []fclient.PublicRoom haveRoomIDs := make(map[string]bool) rand.Shuffle(len(in), func(i, j int) { in[i], in[j] = in[j], in[i] diff --git a/clientapi/routing/directory_public_test.go b/clientapi/routing/directory_public_test.go index 65ad392c2f..de2f01b191 100644 --- a/clientapi/routing/directory_public_test.go +++ b/clientapi/routing/directory_public_test.go @@ -4,17 +4,17 @@ import ( "reflect" "testing" - "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/fclient" ) -func pubRoom(name string) gomatrixserverlib.PublicRoom { - return gomatrixserverlib.PublicRoom{ +func pubRoom(name string) fclient.PublicRoom { + return fclient.PublicRoom{ Name: name, } } func TestSliceInto(t *testing.T) { - slice := []gomatrixserverlib.PublicRoom{ + slice := []fclient.PublicRoom{ pubRoom("a"), pubRoom("b"), pubRoom("c"), pubRoom("d"), pubRoom("e"), pubRoom("f"), pubRoom("g"), } limit := int64(3) @@ -22,7 +22,7 @@ func TestSliceInto(t *testing.T) { since int64 wantPrev int wantNext int - wantSubset []gomatrixserverlib.PublicRoom + wantSubset []fclient.PublicRoom }{ { since: 0, diff --git a/clientapi/routing/login_test.go b/clientapi/routing/login_test.go index b27730767e..bff676826d 100644 --- a/clientapi/routing/login_test.go +++ b/clientapi/routing/login_test.go @@ -17,6 +17,7 @@ import ( "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/jetstream" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/matrix-org/util" "github.com/matrix-org/dendrite/test" @@ -39,7 +40,7 @@ func TestLogin(t *testing.T) { natsInstance := jetstream.NATSInstance{} // add a vhost cfg.Global.VirtualHosts = append(cfg.Global.VirtualHosts, &config.VirtualHost{ - SigningIdentity: gomatrixserverlib.SigningIdentity{ServerName: "vh1"}, + SigningIdentity: fclient.SigningIdentity{ServerName: "vh1"}, }) cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) diff --git a/clientapi/routing/profile.go b/clientapi/routing/profile.go index ab0fd990ef..38b37a25ef 100644 --- a/clientapi/routing/profile.go +++ b/clientapi/routing/profile.go @@ -20,6 +20,7 @@ import ( "time" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/fclient" appserviceAPI "github.com/matrix-org/dendrite/appservice/api" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" @@ -39,7 +40,7 @@ func GetProfile( req *http.Request, profileAPI userapi.ProfileAPI, cfg *config.ClientAPI, userID string, asAPI appserviceAPI.AppServiceInternalAPI, - federation *gomatrixserverlib.FederationClient, + federation *fclient.FederationClient, ) util.JSONResponse { profile, err := getProfile(req.Context(), profileAPI, cfg, userID, asAPI, federation) if err != nil { @@ -67,7 +68,7 @@ func GetProfile( func GetAvatarURL( req *http.Request, profileAPI userapi.ProfileAPI, cfg *config.ClientAPI, userID string, asAPI appserviceAPI.AppServiceInternalAPI, - federation *gomatrixserverlib.FederationClient, + federation *fclient.FederationClient, ) util.JSONResponse { profile := GetProfile(req, profileAPI, cfg, userID, asAPI, federation) p, ok := profile.JSON.(eventutil.UserProfile) @@ -156,7 +157,7 @@ func SetAvatarURL( func GetDisplayName( req *http.Request, profileAPI userapi.ProfileAPI, cfg *config.ClientAPI, userID string, asAPI appserviceAPI.AppServiceInternalAPI, - federation *gomatrixserverlib.FederationClient, + federation *fclient.FederationClient, ) util.JSONResponse { profile := GetProfile(req, profileAPI, cfg, userID, asAPI, federation) p, ok := profile.JSON.(eventutil.UserProfile) @@ -292,7 +293,7 @@ func getProfile( ctx context.Context, profileAPI userapi.ProfileAPI, cfg *config.ClientAPI, userID string, asAPI appserviceAPI.AppServiceInternalAPI, - federation *gomatrixserverlib.FederationClient, + federation *fclient.FederationClient, ) (*authtypes.Profile, error) { localpart, domain, err := gomatrixserverlib.SplitID('@', userID) if err != nil { diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index 246aa9e7fe..c7d01a01d2 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -23,6 +23,7 @@ import ( "github.com/matrix-org/dendrite/setup/base" userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/matrix-org/util" "github.com/nats-io/nats.go" "github.com/prometheus/client_golang/prometheus" @@ -55,7 +56,7 @@ func Setup( asAPI appserviceAPI.AppServiceInternalAPI, userAPI userapi.ClientUserAPI, userDirectoryProvider userapi.QuerySearchProfilesAPI, - federation *gomatrixserverlib.FederationClient, + federation *fclient.FederationClient, syncProducer *producers.SyncAPIProducer, transactionsCache *transactions.Cache, federationSender federationAPI.ClientFederationAPI, diff --git a/clientapi/routing/threepid.go b/clientapi/routing/threepid.go index 8b3548c1fc..102b1d1cbd 100644 --- a/clientapi/routing/threepid.go +++ b/clientapi/routing/threepid.go @@ -24,6 +24,7 @@ import ( "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/userapi/api" userdb "github.com/matrix-org/dendrite/userapi/storage" + "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" @@ -41,7 +42,7 @@ type ThreePIDsResponse struct { // // POST /account/3pid/email/requestToken // POST /register/email/requestToken -func RequestEmailToken(req *http.Request, threePIDAPI api.ClientUserAPI, cfg *config.ClientAPI, client *gomatrixserverlib.Client) util.JSONResponse { +func RequestEmailToken(req *http.Request, threePIDAPI api.ClientUserAPI, cfg *config.ClientAPI, client *fclient.Client) util.JSONResponse { var body threepid.EmailAssociationRequest if reqErr := httputil.UnmarshalJSONRequest(req, &body); reqErr != nil { return *reqErr @@ -92,7 +93,7 @@ func RequestEmailToken(req *http.Request, threePIDAPI api.ClientUserAPI, cfg *co // CheckAndSave3PIDAssociation implements POST /account/3pid func CheckAndSave3PIDAssociation( req *http.Request, threePIDAPI api.ClientUserAPI, device *api.Device, - cfg *config.ClientAPI, client *gomatrixserverlib.Client, + cfg *config.ClientAPI, client *fclient.Client, ) util.JSONResponse { var body threepid.EmailAssociationCheckRequest if reqErr := httputil.UnmarshalJSONRequest(req, &body); reqErr != nil { diff --git a/clientapi/routing/userdirectory.go b/clientapi/routing/userdirectory.go index 62af9efa4f..a4cf8e9c2d 100644 --- a/clientapi/routing/userdirectory.go +++ b/clientapi/routing/userdirectory.go @@ -26,6 +26,7 @@ import ( userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrix" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/matrix-org/util" ) @@ -41,7 +42,7 @@ func SearchUserDirectory( provider userapi.QuerySearchProfilesAPI, searchString string, limit int, - federation *gomatrixserverlib.FederationClient, + federation *fclient.FederationClient, localServerName gomatrixserverlib.ServerName, ) util.JSONResponse { if limit < 10 { diff --git a/clientapi/threepid/threepid.go b/clientapi/threepid/threepid.go index ac4dc38187..1fe573b1b0 100644 --- a/clientapi/threepid/threepid.go +++ b/clientapi/threepid/threepid.go @@ -25,7 +25,7 @@ import ( "strings" "github.com/matrix-org/dendrite/setup/config" - "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/fclient" ) // EmailAssociationRequest represents the request defined at https://matrix.org/docs/spec/client_server/r0.2.0.html#post-matrix-client-r0-register-email-requesttoken @@ -58,7 +58,7 @@ type SID struct { // Returns an error if there was a problem sending the request or decoding the // response, or if the identity server responded with a non-OK status. func CreateSession( - ctx context.Context, req EmailAssociationRequest, cfg *config.ClientAPI, client *gomatrixserverlib.Client, + ctx context.Context, req EmailAssociationRequest, cfg *config.ClientAPI, client *fclient.Client, ) (string, error) { if err := isTrusted(req.IDServer, cfg); err != nil { return "", err @@ -112,7 +112,7 @@ type GetValidatedResponse struct { // response, or if the identity server responded with a non-OK status. func CheckAssociation( ctx context.Context, creds Credentials, cfg *config.ClientAPI, - client *gomatrixserverlib.Client, + client *fclient.Client, ) (bool, string, string, error) { if err := isTrusted(creds.IDServer, cfg); err != nil { return false, "", "", err @@ -146,7 +146,7 @@ func CheckAssociation( // identifier and a Matrix ID. // Returns an error if there was a problem sending the request or decoding the // response, or if the identity server responded with a non-OK status. -func PublishAssociation(ctx context.Context, creds Credentials, userID string, cfg *config.ClientAPI, client *gomatrixserverlib.Client) error { +func PublishAssociation(ctx context.Context, creds Credentials, userID string, cfg *config.ClientAPI, client *fclient.Client) error { if err := isTrusted(creds.IDServer, cfg); err != nil { return err } diff --git a/clientapi/userutil/userutil_test.go b/clientapi/userutil/userutil_test.go index ee6bf8a018..8910983bc4 100644 --- a/clientapi/userutil/userutil_test.go +++ b/clientapi/userutil/userutil_test.go @@ -17,6 +17,7 @@ import ( "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/fclient" ) var ( @@ -30,7 +31,7 @@ var ( // TestGoodUserID checks that correct localpart is returned for a valid user ID. func TestGoodUserID(t *testing.T) { cfg := &config.Global{ - SigningIdentity: gomatrixserverlib.SigningIdentity{ + SigningIdentity: fclient.SigningIdentity{ ServerName: serverName, }, } @@ -49,7 +50,7 @@ func TestGoodUserID(t *testing.T) { // TestWithLocalpartOnly checks that localpart is returned when usernameParam contains only localpart. func TestWithLocalpartOnly(t *testing.T) { cfg := &config.Global{ - SigningIdentity: gomatrixserverlib.SigningIdentity{ + SigningIdentity: fclient.SigningIdentity{ ServerName: serverName, }, } @@ -68,7 +69,7 @@ func TestWithLocalpartOnly(t *testing.T) { // TestIncorrectDomain checks for error when there's server name mismatch. func TestIncorrectDomain(t *testing.T) { cfg := &config.Global{ - SigningIdentity: gomatrixserverlib.SigningIdentity{ + SigningIdentity: fclient.SigningIdentity{ ServerName: invalidServerName, }, } @@ -83,7 +84,7 @@ func TestIncorrectDomain(t *testing.T) { // TestBadUserID checks that ParseUsernameParam fails for invalid user ID func TestBadUserID(t *testing.T) { cfg := &config.Global{ - SigningIdentity: gomatrixserverlib.SigningIdentity{ + SigningIdentity: fclient.SigningIdentity{ ServerName: serverName, }, } diff --git a/cmd/dendrite-demo-pinecone/conn/client.go b/cmd/dendrite-demo-pinecone/conn/client.go index 885de00572..4571de1576 100644 --- a/cmd/dendrite-demo-pinecone/conn/client.go +++ b/cmd/dendrite-demo-pinecone/conn/client.go @@ -22,7 +22,7 @@ import ( "strings" "github.com/matrix-org/dendrite/setup/config" - "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/fclient" "nhooyr.io/websocket" pineconeRouter "github.com/matrix-org/pinecone/router" @@ -91,17 +91,17 @@ func createTransport(s *pineconeSessions.Sessions) *http.Transport { func CreateClient( s *pineconeSessions.Sessions, -) *gomatrixserverlib.Client { - return gomatrixserverlib.NewClient( - gomatrixserverlib.WithTransport(createTransport(s)), +) *fclient.Client { + return fclient.NewClient( + fclient.WithTransport(createTransport(s)), ) } func CreateFederationClient( cfg *config.Dendrite, s *pineconeSessions.Sessions, -) *gomatrixserverlib.FederationClient { - return gomatrixserverlib.NewFederationClient( +) *fclient.FederationClient { + return fclient.NewFederationClient( cfg.Global.SigningIdentities(), - gomatrixserverlib.WithTransport(createTransport(s)), + fclient.WithTransport(createTransport(s)), ) } diff --git a/cmd/dendrite-demo-pinecone/rooms/rooms.go b/cmd/dendrite-demo-pinecone/rooms/rooms.go index 0ac705cc12..808956c97a 100644 --- a/cmd/dendrite-demo-pinecone/rooms/rooms.go +++ b/cmd/dendrite-demo-pinecone/rooms/rooms.go @@ -22,6 +22,7 @@ import ( "github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/defaults" "github.com/matrix-org/dendrite/federationapi/api" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/matrix-org/util" pineconeRouter "github.com/matrix-org/pinecone/router" @@ -32,14 +33,14 @@ type PineconeRoomProvider struct { r *pineconeRouter.Router s *pineconeSessions.Sessions fedSender api.FederationInternalAPI - fedClient *gomatrixserverlib.FederationClient + fedClient *fclient.FederationClient } func NewPineconeRoomProvider( r *pineconeRouter.Router, s *pineconeSessions.Sessions, fedSender api.FederationInternalAPI, - fedClient *gomatrixserverlib.FederationClient, + fedClient *fclient.FederationClient, ) *PineconeRoomProvider { p := &PineconeRoomProvider{ r: r, @@ -50,7 +51,7 @@ func NewPineconeRoomProvider( return p } -func (p *PineconeRoomProvider) Rooms() []gomatrixserverlib.PublicRoom { +func (p *PineconeRoomProvider) Rooms() []fclient.PublicRoom { list := map[gomatrixserverlib.ServerName]struct{}{} for k := range defaults.DefaultServerNames { list[k] = struct{}{} @@ -67,14 +68,14 @@ func (p *PineconeRoomProvider) Rooms() []gomatrixserverlib.PublicRoom { // bulkFetchPublicRoomsFromServers fetches public rooms from the list of homeservers. // Returns a list of public rooms. func bulkFetchPublicRoomsFromServers( - ctx context.Context, fedClient *gomatrixserverlib.FederationClient, + ctx context.Context, fedClient *fclient.FederationClient, origin gomatrixserverlib.ServerName, homeservers map[gomatrixserverlib.ServerName]struct{}, -) (publicRooms []gomatrixserverlib.PublicRoom) { +) (publicRooms []fclient.PublicRoom) { limit := 200 // follow pipeline semantics, see https://blog.golang.org/pipelines for more info. // goroutines send rooms to this channel - roomCh := make(chan gomatrixserverlib.PublicRoom, int(limit)) + roomCh := make(chan fclient.PublicRoom, int(limit)) // signalling channel to tell goroutines to stop sending rooms and quit done := make(chan bool) // signalling to say when we can close the room channel diff --git a/cmd/dendrite-demo-pinecone/users/users.go b/cmd/dendrite-demo-pinecone/users/users.go index fc66bf2996..5bd056e186 100644 --- a/cmd/dendrite-demo-pinecone/users/users.go +++ b/cmd/dendrite-demo-pinecone/users/users.go @@ -28,6 +28,7 @@ import ( "github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/defaults" userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/matrix-org/util" pineconeRouter "github.com/matrix-org/pinecone/router" @@ -38,7 +39,7 @@ type PineconeUserProvider struct { r *pineconeRouter.Router s *pineconeSessions.Sessions userAPI userapi.QuerySearchProfilesAPI - fedClient *gomatrixserverlib.FederationClient + fedClient *fclient.FederationClient } const PublicURL = "/_matrix/p2p/profiles" @@ -47,7 +48,7 @@ func NewPineconeUserProvider( r *pineconeRouter.Router, s *pineconeSessions.Sessions, userAPI userapi.QuerySearchProfilesAPI, - fedClient *gomatrixserverlib.FederationClient, + fedClient *fclient.FederationClient, ) *PineconeUserProvider { p := &PineconeUserProvider{ r: r, @@ -94,7 +95,7 @@ func (p *PineconeUserProvider) QuerySearchProfiles(ctx context.Context, req *use // Returns a list of user profiles. func bulkFetchUserDirectoriesFromServers( ctx context.Context, req *userapi.QuerySearchProfilesRequest, - fedClient *gomatrixserverlib.FederationClient, + fedClient *fclient.FederationClient, homeservers map[gomatrixserverlib.ServerName]struct{}, ) (profiles []authtypes.Profile) { jsonBody, err := json.Marshal(req) diff --git a/cmd/dendrite-demo-yggdrasil/yggconn/client.go b/cmd/dendrite-demo-yggdrasil/yggconn/client.go index 51365547b7..c25acf2ec7 100644 --- a/cmd/dendrite-demo-yggdrasil/yggconn/client.go +++ b/cmd/dendrite-demo-yggdrasil/yggconn/client.go @@ -5,7 +5,7 @@ import ( "time" "github.com/matrix-org/dendrite/setup/config" - "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/fclient" ) type yggroundtripper struct { @@ -17,7 +17,7 @@ func (y *yggroundtripper) RoundTrip(req *http.Request) (*http.Response, error) { return y.inner.RoundTrip(req) } -func (n *Node) CreateClient() *gomatrixserverlib.Client { +func (n *Node) CreateClient() *fclient.Client { tr := &http.Transport{} tr.RegisterProtocol( "matrix", &yggroundtripper{ @@ -31,14 +31,14 @@ func (n *Node) CreateClient() *gomatrixserverlib.Client { }, }, ) - return gomatrixserverlib.NewClient( - gomatrixserverlib.WithTransport(tr), + return fclient.NewClient( + fclient.WithTransport(tr), ) } func (n *Node) CreateFederationClient( cfg *config.Dendrite, -) *gomatrixserverlib.FederationClient { +) *fclient.FederationClient { tr := &http.Transport{} tr.RegisterProtocol( "matrix", &yggroundtripper{ @@ -52,8 +52,8 @@ func (n *Node) CreateFederationClient( }, }, ) - return gomatrixserverlib.NewFederationClient( + return fclient.NewFederationClient( cfg.Global.SigningIdentities(), - gomatrixserverlib.WithTransport(tr), + fclient.WithTransport(tr), ) } diff --git a/cmd/dendrite-demo-yggdrasil/yggrooms/yggrooms.go b/cmd/dendrite-demo-yggdrasil/yggrooms/yggrooms.go index 0de64755e7..180990d54d 100644 --- a/cmd/dendrite-demo-yggdrasil/yggrooms/yggrooms.go +++ b/cmd/dendrite-demo-yggdrasil/yggrooms/yggrooms.go @@ -22,17 +22,18 @@ import ( "github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/yggconn" "github.com/matrix-org/dendrite/federationapi/api" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/matrix-org/util" ) type YggdrasilRoomProvider struct { node *yggconn.Node fedSender api.FederationInternalAPI - fedClient *gomatrixserverlib.FederationClient + fedClient *fclient.FederationClient } func NewYggdrasilRoomProvider( - node *yggconn.Node, fedSender api.FederationInternalAPI, fedClient *gomatrixserverlib.FederationClient, + node *yggconn.Node, fedSender api.FederationInternalAPI, fedClient *fclient.FederationClient, ) *YggdrasilRoomProvider { p := &YggdrasilRoomProvider{ node: node, @@ -42,7 +43,7 @@ func NewYggdrasilRoomProvider( return p } -func (p *YggdrasilRoomProvider) Rooms() []gomatrixserverlib.PublicRoom { +func (p *YggdrasilRoomProvider) Rooms() []fclient.PublicRoom { return bulkFetchPublicRoomsFromServers( context.Background(), p.fedClient, gomatrixserverlib.ServerName(p.node.DerivedServerName()), @@ -53,14 +54,14 @@ func (p *YggdrasilRoomProvider) Rooms() []gomatrixserverlib.PublicRoom { // bulkFetchPublicRoomsFromServers fetches public rooms from the list of homeservers. // Returns a list of public rooms. func bulkFetchPublicRoomsFromServers( - ctx context.Context, fedClient *gomatrixserverlib.FederationClient, + ctx context.Context, fedClient *fclient.FederationClient, origin gomatrixserverlib.ServerName, homeservers []gomatrixserverlib.ServerName, -) (publicRooms []gomatrixserverlib.PublicRoom) { +) (publicRooms []fclient.PublicRoom) { limit := 200 // follow pipeline semantics, see https://blog.golang.org/pipelines for more info. // goroutines send rooms to this channel - roomCh := make(chan gomatrixserverlib.PublicRoom, int(limit)) + roomCh := make(chan fclient.PublicRoom, int(limit)) // signalling channel to tell goroutines to stop sending rooms and quit done := make(chan bool) // signalling to say when we can close the room channel diff --git a/cmd/dendrite/main.go b/cmd/dendrite/main.go index a22677ebe0..66eb88f875 100644 --- a/cmd/dendrite/main.go +++ b/cmd/dendrite/main.go @@ -25,7 +25,7 @@ import ( "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/setup/jetstream" "github.com/matrix-org/dendrite/setup/process" - "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/sirupsen/logrus" "github.com/matrix-org/dendrite/appservice" @@ -96,9 +96,9 @@ func main() { } // create DNS cache - var dnsCache *gomatrixserverlib.DNSCache + var dnsCache *fclient.DNSCache if cfg.Global.DNSCache.Enabled { - dnsCache = gomatrixserverlib.NewDNSCache( + dnsCache = fclient.NewDNSCache( cfg.Global.DNSCache.CacheSize, cfg.Global.DNSCache.CacheLifetime, ) diff --git a/cmd/furl/main.go b/cmd/furl/main.go index b208ba8680..32e9970495 100644 --- a/cmd/furl/main.go +++ b/cmd/furl/main.go @@ -13,6 +13,7 @@ import ( "os" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/fclient" ) var requestFrom = flag.String("from", "", "the server name that the request should originate from") @@ -49,8 +50,8 @@ func main() { } serverName := gomatrixserverlib.ServerName(*requestFrom) - client := gomatrixserverlib.NewFederationClient( - []*gomatrixserverlib.SigningIdentity{ + client := fclient.NewFederationClient( + []*fclient.SigningIdentity{ { ServerName: serverName, KeyID: gomatrixserverlib.KeyID(keyBlock.Headers["Key-ID"]), diff --git a/federationapi/api/api.go b/federationapi/api/api.go index e4c0b27148..e23bec2714 100644 --- a/federationapi/api/api.go +++ b/federationapi/api/api.go @@ -7,6 +7,7 @@ import ( "github.com/matrix-org/gomatrix" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/matrix-org/dendrite/federationapi/types" ) @@ -22,8 +23,8 @@ type FederationInternalAPI interface { QueryServerKeys(ctx context.Context, request *QueryServerKeysRequest, response *QueryServerKeysResponse) error LookupServerKeys(ctx context.Context, s gomatrixserverlib.ServerName, keyRequests map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp) ([]gomatrixserverlib.ServerKeys, error) - MSC2836EventRelationships(ctx context.Context, origin, dst gomatrixserverlib.ServerName, r gomatrixserverlib.MSC2836EventRelationshipsRequest, roomVersion gomatrixserverlib.RoomVersion) (res gomatrixserverlib.MSC2836EventRelationshipsResponse, err error) - MSC2946Spaces(ctx context.Context, origin, dst gomatrixserverlib.ServerName, roomID string, suggestedOnly bool) (res gomatrixserverlib.MSC2946SpacesResponse, err error) + MSC2836EventRelationships(ctx context.Context, origin, dst gomatrixserverlib.ServerName, r fclient.MSC2836EventRelationshipsRequest, roomVersion gomatrixserverlib.RoomVersion) (res fclient.MSC2836EventRelationshipsResponse, err error) + MSC2946Spaces(ctx context.Context, origin, dst gomatrixserverlib.ServerName, roomID string, suggestedOnly bool) (res fclient.MSC2946SpacesResponse, err error) // Broadcasts an EDU to all servers in rooms we are joined to. Used in the yggdrasil demos. PerformBroadcastEDU( @@ -66,9 +67,9 @@ type RoomserverFederationAPI interface { // containing only the server names (without information for membership events). // The response will include this server if they are joined to the room. QueryJoinedHostServerNamesInRoom(ctx context.Context, request *QueryJoinedHostServerNamesInRoomRequest, response *QueryJoinedHostServerNamesInRoomResponse) error - GetEventAuth(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomVersion gomatrixserverlib.RoomVersion, roomID, eventID string) (res gomatrixserverlib.RespEventAuth, err error) + GetEventAuth(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomVersion gomatrixserverlib.RoomVersion, roomID, eventID string) (res fclient.RespEventAuth, err error) GetEvent(ctx context.Context, origin, s gomatrixserverlib.ServerName, eventID string) (res gomatrixserverlib.Transaction, err error) - LookupMissingEvents(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID string, missing gomatrixserverlib.MissingEvents, roomVersion gomatrixserverlib.RoomVersion) (res gomatrixserverlib.RespMissingEvents, err error) + LookupMissingEvents(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID string, missing fclient.MissingEvents, roomVersion gomatrixserverlib.RoomVersion) (res fclient.RespMissingEvents, err error) } type P2PFederationAPI interface { @@ -98,45 +99,45 @@ type P2PFederationAPI interface { // implements as proxy calls, with built-in backoff/retries/etc. Errors returned from functions in // this interface are of type FederationClientError type KeyserverFederationAPI interface { - GetUserDevices(ctx context.Context, origin, s gomatrixserverlib.ServerName, userID string) (res gomatrixserverlib.RespUserDevices, err error) - ClaimKeys(ctx context.Context, origin, s gomatrixserverlib.ServerName, oneTimeKeys map[string]map[string]string) (res gomatrixserverlib.RespClaimKeys, err error) - QueryKeys(ctx context.Context, origin, s gomatrixserverlib.ServerName, keys map[string][]string) (res gomatrixserverlib.RespQueryKeys, err error) + GetUserDevices(ctx context.Context, origin, s gomatrixserverlib.ServerName, userID string) (res fclient.RespUserDevices, err error) + ClaimKeys(ctx context.Context, origin, s gomatrixserverlib.ServerName, oneTimeKeys map[string]map[string]string) (res fclient.RespClaimKeys, err error) + QueryKeys(ctx context.Context, origin, s gomatrixserverlib.ServerName, keys map[string][]string) (res fclient.RespQueryKeys, err error) } // an interface for gmsl.FederationClient - contains functions called by federationapi only. type FederationClient interface { P2PFederationClient gomatrixserverlib.KeyClient - SendTransaction(ctx context.Context, t gomatrixserverlib.Transaction) (res gomatrixserverlib.RespSend, err error) + SendTransaction(ctx context.Context, t gomatrixserverlib.Transaction) (res fclient.RespSend, err error) // Perform operations - LookupRoomAlias(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomAlias string) (res gomatrixserverlib.RespDirectory, err error) - Peek(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID, peekID string, roomVersions []gomatrixserverlib.RoomVersion) (res gomatrixserverlib.RespPeek, err error) - MakeJoin(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID, userID string, roomVersions []gomatrixserverlib.RoomVersion) (res gomatrixserverlib.RespMakeJoin, err error) - SendJoin(ctx context.Context, origin, s gomatrixserverlib.ServerName, event *gomatrixserverlib.Event) (res gomatrixserverlib.RespSendJoin, err error) - MakeLeave(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID, userID string) (res gomatrixserverlib.RespMakeLeave, err error) + LookupRoomAlias(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomAlias string) (res fclient.RespDirectory, err error) + Peek(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID, peekID string, roomVersions []gomatrixserverlib.RoomVersion) (res fclient.RespPeek, err error) + MakeJoin(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID, userID string, roomVersions []gomatrixserverlib.RoomVersion) (res fclient.RespMakeJoin, err error) + SendJoin(ctx context.Context, origin, s gomatrixserverlib.ServerName, event *gomatrixserverlib.Event) (res fclient.RespSendJoin, err error) + MakeLeave(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID, userID string) (res fclient.RespMakeLeave, err error) SendLeave(ctx context.Context, origin, s gomatrixserverlib.ServerName, event *gomatrixserverlib.Event) (err error) - SendInviteV2(ctx context.Context, origin, s gomatrixserverlib.ServerName, request gomatrixserverlib.InviteV2Request) (res gomatrixserverlib.RespInviteV2, err error) + SendInviteV2(ctx context.Context, origin, s gomatrixserverlib.ServerName, request gomatrixserverlib.InviteV2Request) (res fclient.RespInviteV2, err error) GetEvent(ctx context.Context, origin, s gomatrixserverlib.ServerName, eventID string) (res gomatrixserverlib.Transaction, err error) - GetEventAuth(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomVersion gomatrixserverlib.RoomVersion, roomID, eventID string) (res gomatrixserverlib.RespEventAuth, err error) - GetUserDevices(ctx context.Context, origin, s gomatrixserverlib.ServerName, userID string) (gomatrixserverlib.RespUserDevices, error) - ClaimKeys(ctx context.Context, origin, s gomatrixserverlib.ServerName, oneTimeKeys map[string]map[string]string) (gomatrixserverlib.RespClaimKeys, error) - QueryKeys(ctx context.Context, origin, s gomatrixserverlib.ServerName, keys map[string][]string) (gomatrixserverlib.RespQueryKeys, error) + GetEventAuth(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomVersion gomatrixserverlib.RoomVersion, roomID, eventID string) (res fclient.RespEventAuth, err error) + GetUserDevices(ctx context.Context, origin, s gomatrixserverlib.ServerName, userID string) (fclient.RespUserDevices, error) + ClaimKeys(ctx context.Context, origin, s gomatrixserverlib.ServerName, oneTimeKeys map[string]map[string]string) (fclient.RespClaimKeys, error) + QueryKeys(ctx context.Context, origin, s gomatrixserverlib.ServerName, keys map[string][]string) (fclient.RespQueryKeys, error) Backfill(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID string, limit int, eventIDs []string) (res gomatrixserverlib.Transaction, err error) - MSC2836EventRelationships(ctx context.Context, origin, dst gomatrixserverlib.ServerName, r gomatrixserverlib.MSC2836EventRelationshipsRequest, roomVersion gomatrixserverlib.RoomVersion) (res gomatrixserverlib.MSC2836EventRelationshipsResponse, err error) - MSC2946Spaces(ctx context.Context, origin, dst gomatrixserverlib.ServerName, roomID string, suggestedOnly bool) (res gomatrixserverlib.MSC2946SpacesResponse, err error) + MSC2836EventRelationships(ctx context.Context, origin, dst gomatrixserverlib.ServerName, r fclient.MSC2836EventRelationshipsRequest, roomVersion gomatrixserverlib.RoomVersion) (res fclient.MSC2836EventRelationshipsResponse, err error) + MSC2946Spaces(ctx context.Context, origin, dst gomatrixserverlib.ServerName, roomID string, suggestedOnly bool) (res fclient.MSC2946SpacesResponse, err error) ExchangeThirdPartyInvite(ctx context.Context, origin, s gomatrixserverlib.ServerName, builder gomatrixserverlib.EventBuilder) (err error) - LookupState(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID string, eventID string, roomVersion gomatrixserverlib.RoomVersion) (res gomatrixserverlib.RespState, err error) - LookupStateIDs(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID string, eventID string) (res gomatrixserverlib.RespStateIDs, err error) - LookupMissingEvents(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID string, missing gomatrixserverlib.MissingEvents, roomVersion gomatrixserverlib.RoomVersion) (res gomatrixserverlib.RespMissingEvents, err error) + LookupState(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID string, eventID string, roomVersion gomatrixserverlib.RoomVersion) (res fclient.RespState, err error) + LookupStateIDs(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID string, eventID string) (res fclient.RespStateIDs, err error) + LookupMissingEvents(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID string, missing fclient.MissingEvents, roomVersion gomatrixserverlib.RoomVersion) (res fclient.RespMissingEvents, err error) } type P2PFederationClient interface { - P2PSendTransactionToRelay(ctx context.Context, u gomatrixserverlib.UserID, t gomatrixserverlib.Transaction, forwardingServer gomatrixserverlib.ServerName) (res gomatrixserverlib.EmptyResp, err error) - P2PGetTransactionFromRelay(ctx context.Context, u gomatrixserverlib.UserID, prev gomatrixserverlib.RelayEntry, relayServer gomatrixserverlib.ServerName) (res gomatrixserverlib.RespGetRelayTransaction, err error) + P2PSendTransactionToRelay(ctx context.Context, u gomatrixserverlib.UserID, t gomatrixserverlib.Transaction, forwardingServer gomatrixserverlib.ServerName) (res fclient.EmptyResp, err error) + P2PGetTransactionFromRelay(ctx context.Context, u gomatrixserverlib.UserID, prev fclient.RelayEntry, relayServer gomatrixserverlib.ServerName) (res fclient.RespGetRelayTransaction, err error) } // FederationClientError is returned from FederationClient methods in the event of a problem. diff --git a/federationapi/federationapi.go b/federationapi/federationapi.go index 3b5394a136..c64fa550de 100644 --- a/federationapi/federationapi.go +++ b/federationapi/federationapi.go @@ -21,6 +21,7 @@ import ( "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/process" + "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/sirupsen/logrus" "github.com/matrix-org/dendrite/federationapi/api" @@ -48,7 +49,7 @@ func AddPublicRoutes( dendriteConfig *config.Dendrite, natsInstance *jetstream.NATSInstance, userAPI userapi.FederationUserAPI, - federation *gomatrixserverlib.FederationClient, + federation *fclient.FederationClient, keyRing gomatrixserverlib.JSONVerifier, rsAPI roomserverAPI.FederationRoomserverAPI, fedAPI federationAPI.FederationInternalAPI, diff --git a/federationapi/federationapi_keys_test.go b/federationapi/federationapi_keys_test.go index 425cdbb972..2fa748bade 100644 --- a/federationapi/federationapi_keys_test.go +++ b/federationapi/federationapi_keys_test.go @@ -16,6 +16,7 @@ import ( "github.com/matrix-org/dendrite/setup/jetstream" "github.com/matrix-org/dendrite/setup/process" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/matrix-org/dendrite/federationapi/api" "github.com/matrix-org/dendrite/federationapi/routing" @@ -24,12 +25,12 @@ import ( ) type server struct { - name gomatrixserverlib.ServerName // server name - validity time.Duration // key validity duration from now - config *config.FederationAPI // skeleton config, from TestMain - fedclient *gomatrixserverlib.FederationClient // uses MockRoundTripper - cache *caching.Caches // server-specific cache - api api.FederationInternalAPI // server-specific server key API + name gomatrixserverlib.ServerName // server name + validity time.Duration // key validity duration from now + config *config.FederationAPI // skeleton config, from TestMain + fedclient *fclient.FederationClient // uses MockRoundTripper + cache *caching.Caches // server-specific cache + api api.FederationInternalAPI // server-specific server key API } func (s *server) renew() { @@ -105,9 +106,9 @@ func TestMain(m *testing.M) { transport.RegisterProtocol("matrix", &MockRoundTripper{}) // Create the federation client. - s.fedclient = gomatrixserverlib.NewFederationClient( + s.fedclient = fclient.NewFederationClient( s.config.Matrix.SigningIdentities(), - gomatrixserverlib.WithTransport(transport), + fclient.WithTransport(transport), ) // Finally, build the server key APIs. diff --git a/federationapi/federationapi_test.go b/federationapi/federationapi_test.go index d39f40512c..3c01a82596 100644 --- a/federationapi/federationapi_test.go +++ b/federationapi/federationapi_test.go @@ -15,6 +15,7 @@ import ( "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrix" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/nats-io/nats.go" "github.com/matrix-org/dendrite/federationapi" @@ -104,7 +105,7 @@ func (f *fedClient) GetServerKeys(ctx context.Context, matrixServer gomatrixserv return keys, nil } -func (f *fedClient) MakeJoin(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID, userID string, roomVersions []gomatrixserverlib.RoomVersion) (res gomatrixserverlib.RespMakeJoin, err error) { +func (f *fedClient) MakeJoin(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID, userID string, roomVersions []gomatrixserverlib.RoomVersion) (res fclient.RespMakeJoin, err error) { for _, r := range f.allowJoins { if r.ID == roomID { res.RoomVersion = r.Version @@ -128,7 +129,7 @@ func (f *fedClient) MakeJoin(ctx context.Context, origin, s gomatrixserverlib.Se } return } -func (f *fedClient) SendJoin(ctx context.Context, origin, s gomatrixserverlib.ServerName, event *gomatrixserverlib.Event) (res gomatrixserverlib.RespSendJoin, err error) { +func (f *fedClient) SendJoin(ctx context.Context, origin, s gomatrixserverlib.ServerName, event *gomatrixserverlib.Event) (res fclient.RespSendJoin, err error) { f.fedClientMutex.Lock() defer f.fedClientMutex.Unlock() for _, r := range f.allowJoins { @@ -142,7 +143,7 @@ func (f *fedClient) SendJoin(ctx context.Context, origin, s gomatrixserverlib.Se return } -func (f *fedClient) SendTransaction(ctx context.Context, t gomatrixserverlib.Transaction) (res gomatrixserverlib.RespSend, err error) { +func (f *fedClient) SendTransaction(ctx context.Context, t gomatrixserverlib.Transaction) (res fclient.RespSend, err error) { f.fedClientMutex.Lock() defer f.fedClientMutex.Unlock() for _, edu := range t.EDUs { @@ -313,9 +314,9 @@ func TestRoomsV3URLEscapeDoNot404(t *testing.T) { defer cancel() serverName := gomatrixserverlib.ServerName(strings.TrimPrefix(baseURL, "https://")) - fedCli := gomatrixserverlib.NewFederationClient( + fedCli := fclient.NewFederationClient( cfg.Global.SigningIdentities(), - gomatrixserverlib.WithSkipVerify(true), + fclient.WithSkipVerify(true), ) for _, tc := range testCases { diff --git a/federationapi/internal/federationclient.go b/federationapi/internal/federationclient.go index db6348ec17..b0d5b1d1f2 100644 --- a/federationapi/internal/federationclient.go +++ b/federationapi/internal/federationclient.go @@ -5,6 +5,7 @@ import ( "time" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/fclient" ) // Functions here are "proxying" calls to the gomatrixserverlib federation @@ -13,56 +14,56 @@ import ( func (a *FederationInternalAPI) GetEventAuth( ctx context.Context, origin, s gomatrixserverlib.ServerName, roomVersion gomatrixserverlib.RoomVersion, roomID, eventID string, -) (res gomatrixserverlib.RespEventAuth, err error) { +) (res fclient.RespEventAuth, err error) { ctx, cancel := context.WithTimeout(ctx, time.Second*30) defer cancel() ires, err := a.doRequestIfNotBlacklisted(s, func() (interface{}, error) { return a.federation.GetEventAuth(ctx, origin, s, roomVersion, roomID, eventID) }) if err != nil { - return gomatrixserverlib.RespEventAuth{}, err + return fclient.RespEventAuth{}, err } - return ires.(gomatrixserverlib.RespEventAuth), nil + return ires.(fclient.RespEventAuth), nil } func (a *FederationInternalAPI) GetUserDevices( ctx context.Context, origin, s gomatrixserverlib.ServerName, userID string, -) (gomatrixserverlib.RespUserDevices, error) { +) (fclient.RespUserDevices, error) { ctx, cancel := context.WithTimeout(ctx, time.Second*30) defer cancel() ires, err := a.doRequestIfNotBlacklisted(s, func() (interface{}, error) { return a.federation.GetUserDevices(ctx, origin, s, userID) }) if err != nil { - return gomatrixserverlib.RespUserDevices{}, err + return fclient.RespUserDevices{}, err } - return ires.(gomatrixserverlib.RespUserDevices), nil + return ires.(fclient.RespUserDevices), nil } func (a *FederationInternalAPI) ClaimKeys( ctx context.Context, origin, s gomatrixserverlib.ServerName, oneTimeKeys map[string]map[string]string, -) (gomatrixserverlib.RespClaimKeys, error) { +) (fclient.RespClaimKeys, error) { ctx, cancel := context.WithTimeout(ctx, time.Second*30) defer cancel() ires, err := a.doRequestIfNotBlacklisted(s, func() (interface{}, error) { return a.federation.ClaimKeys(ctx, origin, s, oneTimeKeys) }) if err != nil { - return gomatrixserverlib.RespClaimKeys{}, err + return fclient.RespClaimKeys{}, err } - return ires.(gomatrixserverlib.RespClaimKeys), nil + return ires.(fclient.RespClaimKeys), nil } func (a *FederationInternalAPI) QueryKeys( ctx context.Context, origin, s gomatrixserverlib.ServerName, keys map[string][]string, -) (gomatrixserverlib.RespQueryKeys, error) { +) (fclient.RespQueryKeys, error) { ires, err := a.doRequestIfNotBackingOffOrBlacklisted(s, func() (interface{}, error) { return a.federation.QueryKeys(ctx, origin, s, keys) }) if err != nil { - return gomatrixserverlib.RespQueryKeys{}, err + return fclient.RespQueryKeys{}, err } - return ires.(gomatrixserverlib.RespQueryKeys), nil + return ires.(fclient.RespQueryKeys), nil } func (a *FederationInternalAPI) Backfill( @@ -81,45 +82,46 @@ func (a *FederationInternalAPI) Backfill( func (a *FederationInternalAPI) LookupState( ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID, eventID string, roomVersion gomatrixserverlib.RoomVersion, -) (res gomatrixserverlib.RespState, err error) { +) (res gomatrixserverlib.StateResponse, err error) { ctx, cancel := context.WithTimeout(ctx, time.Second*30) defer cancel() ires, err := a.doRequestIfNotBlacklisted(s, func() (interface{}, error) { return a.federation.LookupState(ctx, origin, s, roomID, eventID, roomVersion) }) if err != nil { - return gomatrixserverlib.RespState{}, err + return &fclient.RespState{}, err } - return ires.(gomatrixserverlib.RespState), nil + r := ires.(fclient.RespState) + return &r, nil } func (a *FederationInternalAPI) LookupStateIDs( ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID, eventID string, -) (res gomatrixserverlib.RespStateIDs, err error) { +) (res gomatrixserverlib.StateIDResponse, err error) { ctx, cancel := context.WithTimeout(ctx, time.Second*30) defer cancel() ires, err := a.doRequestIfNotBlacklisted(s, func() (interface{}, error) { return a.federation.LookupStateIDs(ctx, origin, s, roomID, eventID) }) if err != nil { - return gomatrixserverlib.RespStateIDs{}, err + return fclient.RespStateIDs{}, err } - return ires.(gomatrixserverlib.RespStateIDs), nil + return ires.(fclient.RespStateIDs), nil } func (a *FederationInternalAPI) LookupMissingEvents( ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID string, - missing gomatrixserverlib.MissingEvents, roomVersion gomatrixserverlib.RoomVersion, -) (res gomatrixserverlib.RespMissingEvents, err error) { + missing fclient.MissingEvents, roomVersion gomatrixserverlib.RoomVersion, +) (res fclient.RespMissingEvents, err error) { ctx, cancel := context.WithTimeout(ctx, time.Second*30) defer cancel() ires, err := a.doRequestIfNotBlacklisted(s, func() (interface{}, error) { return a.federation.LookupMissingEvents(ctx, origin, s, roomID, missing, roomVersion) }) if err != nil { - return gomatrixserverlib.RespMissingEvents{}, err + return fclient.RespMissingEvents{}, err } - return ires.(gomatrixserverlib.RespMissingEvents), nil + return ires.(fclient.RespMissingEvents), nil } func (a *FederationInternalAPI) GetEvent( @@ -151,9 +153,9 @@ func (a *FederationInternalAPI) LookupServerKeys( } func (a *FederationInternalAPI) MSC2836EventRelationships( - ctx context.Context, origin, s gomatrixserverlib.ServerName, r gomatrixserverlib.MSC2836EventRelationshipsRequest, + ctx context.Context, origin, s gomatrixserverlib.ServerName, r fclient.MSC2836EventRelationshipsRequest, roomVersion gomatrixserverlib.RoomVersion, -) (res gomatrixserverlib.MSC2836EventRelationshipsResponse, err error) { +) (res fclient.MSC2836EventRelationshipsResponse, err error) { ctx, cancel := context.WithTimeout(ctx, time.Minute) defer cancel() ires, err := a.doRequestIfNotBlacklisted(s, func() (interface{}, error) { @@ -162,12 +164,12 @@ func (a *FederationInternalAPI) MSC2836EventRelationships( if err != nil { return res, err } - return ires.(gomatrixserverlib.MSC2836EventRelationshipsResponse), nil + return ires.(fclient.MSC2836EventRelationshipsResponse), nil } func (a *FederationInternalAPI) MSC2946Spaces( ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID string, suggestedOnly bool, -) (res gomatrixserverlib.MSC2946SpacesResponse, err error) { +) (res fclient.MSC2946SpacesResponse, err error) { ctx, cancel := context.WithTimeout(ctx, time.Minute) defer cancel() ires, err := a.doRequestIfNotBlacklisted(s, func() (interface{}, error) { @@ -176,5 +178,5 @@ func (a *FederationInternalAPI) MSC2946Spaces( if err != nil { return res, err } - return ires.(gomatrixserverlib.MSC2946SpacesResponse), nil + return ires.(fclient.MSC2946SpacesResponse), nil } diff --git a/federationapi/internal/federationclient_test.go b/federationapi/internal/federationclient_test.go index 49137e2d8c..948a96eec1 100644 --- a/federationapi/internal/federationclient_test.go +++ b/federationapi/internal/federationclient_test.go @@ -25,6 +25,7 @@ import ( "github.com/matrix-org/dendrite/setup/process" "github.com/matrix-org/dendrite/test" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/stretchr/testify/assert" ) @@ -33,20 +34,20 @@ const ( FailuresUntilBlacklist = 8 ) -func (t *testFedClient) QueryKeys(ctx context.Context, origin, s gomatrixserverlib.ServerName, keys map[string][]string) (gomatrixserverlib.RespQueryKeys, error) { +func (t *testFedClient) QueryKeys(ctx context.Context, origin, s gomatrixserverlib.ServerName, keys map[string][]string) (fclient.RespQueryKeys, error) { t.queryKeysCalled = true if t.shouldFail { - return gomatrixserverlib.RespQueryKeys{}, fmt.Errorf("Failure") + return fclient.RespQueryKeys{}, fmt.Errorf("Failure") } - return gomatrixserverlib.RespQueryKeys{}, nil + return fclient.RespQueryKeys{}, nil } -func (t *testFedClient) ClaimKeys(ctx context.Context, origin, s gomatrixserverlib.ServerName, oneTimeKeys map[string]map[string]string) (gomatrixserverlib.RespClaimKeys, error) { +func (t *testFedClient) ClaimKeys(ctx context.Context, origin, s gomatrixserverlib.ServerName, oneTimeKeys map[string]map[string]string) (fclient.RespClaimKeys, error) { t.claimKeysCalled = true if t.shouldFail { - return gomatrixserverlib.RespClaimKeys{}, fmt.Errorf("Failure") + return fclient.RespClaimKeys{}, fmt.Errorf("Failure") } - return gomatrixserverlib.RespClaimKeys{}, nil + return fclient.RespClaimKeys{}, nil } func TestFederationClientQueryKeys(t *testing.T) { @@ -54,7 +55,7 @@ func TestFederationClientQueryKeys(t *testing.T) { cfg := config.FederationAPI{ Matrix: &config.Global{ - SigningIdentity: gomatrixserverlib.SigningIdentity{ + SigningIdentity: fclient.SigningIdentity{ ServerName: "server", }, }, @@ -85,7 +86,7 @@ func TestFederationClientQueryKeysBlacklisted(t *testing.T) { cfg := config.FederationAPI{ Matrix: &config.Global{ - SigningIdentity: gomatrixserverlib.SigningIdentity{ + SigningIdentity: fclient.SigningIdentity{ ServerName: "server", }, }, @@ -115,7 +116,7 @@ func TestFederationClientQueryKeysFailure(t *testing.T) { cfg := config.FederationAPI{ Matrix: &config.Global{ - SigningIdentity: gomatrixserverlib.SigningIdentity{ + SigningIdentity: fclient.SigningIdentity{ ServerName: "server", }, }, @@ -145,7 +146,7 @@ func TestFederationClientClaimKeys(t *testing.T) { cfg := config.FederationAPI{ Matrix: &config.Global{ - SigningIdentity: gomatrixserverlib.SigningIdentity{ + SigningIdentity: fclient.SigningIdentity{ ServerName: "server", }, }, @@ -176,7 +177,7 @@ func TestFederationClientClaimKeysBlacklisted(t *testing.T) { cfg := config.FederationAPI{ Matrix: &config.Global{ - SigningIdentity: gomatrixserverlib.SigningIdentity{ + SigningIdentity: fclient.SigningIdentity{ ServerName: "server", }, }, diff --git a/federationapi/internal/perform.go b/federationapi/internal/perform.go index dadb2b2b30..08287c6921 100644 --- a/federationapi/internal/perform.go +++ b/federationapi/internal/perform.go @@ -9,6 +9,7 @@ import ( "github.com/matrix-org/gomatrix" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/matrix-org/util" "github.com/sirupsen/logrus" @@ -255,7 +256,7 @@ func (r *FederationInternalAPI) performJoinUsingServer( // waste the effort. // TODO: Can we expand Check here to return a list of missing auth // events rather than failing one at a time? - var respState *gomatrixserverlib.RespState + var respState *fclient.RespState respState, err = respSendJoin.Check( context.Background(), respMakeJoin.RoomVersion, diff --git a/federationapi/internal/perform_test.go b/federationapi/internal/perform_test.go index e6e366f996..90849dcf62 100644 --- a/federationapi/internal/perform_test.go +++ b/federationapi/internal/perform_test.go @@ -25,6 +25,7 @@ import ( "github.com/matrix-org/dendrite/setup/process" "github.com/matrix-org/dendrite/test" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/stretchr/testify/assert" ) @@ -35,8 +36,8 @@ type testFedClient struct { shouldFail bool } -func (t *testFedClient) LookupRoomAlias(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomAlias string) (res gomatrixserverlib.RespDirectory, err error) { - return gomatrixserverlib.RespDirectory{}, nil +func (t *testFedClient) LookupRoomAlias(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomAlias string) (res fclient.RespDirectory, err error) { + return fclient.RespDirectory{}, nil } func TestPerformWakeupServers(t *testing.T) { @@ -54,7 +55,7 @@ func TestPerformWakeupServers(t *testing.T) { cfg := config.FederationAPI{ Matrix: &config.Global{ - SigningIdentity: gomatrixserverlib.SigningIdentity{ + SigningIdentity: fclient.SigningIdentity{ ServerName: "relay", }, }, @@ -96,7 +97,7 @@ func TestQueryRelayServers(t *testing.T) { cfg := config.FederationAPI{ Matrix: &config.Global{ - SigningIdentity: gomatrixserverlib.SigningIdentity{ + SigningIdentity: fclient.SigningIdentity{ ServerName: "relay", }, }, @@ -133,7 +134,7 @@ func TestRemoveRelayServers(t *testing.T) { cfg := config.FederationAPI{ Matrix: &config.Global{ - SigningIdentity: gomatrixserverlib.SigningIdentity{ + SigningIdentity: fclient.SigningIdentity{ ServerName: "relay", }, }, @@ -169,7 +170,7 @@ func TestPerformDirectoryLookup(t *testing.T) { cfg := config.FederationAPI{ Matrix: &config.Global{ - SigningIdentity: gomatrixserverlib.SigningIdentity{ + SigningIdentity: fclient.SigningIdentity{ ServerName: "relay", }, }, @@ -204,7 +205,7 @@ func TestPerformDirectoryLookupRelaying(t *testing.T) { cfg := config.FederationAPI{ Matrix: &config.Global{ - SigningIdentity: gomatrixserverlib.SigningIdentity{ + SigningIdentity: fclient.SigningIdentity{ ServerName: server, }, }, diff --git a/federationapi/queue/destinationqueue.go b/federationapi/queue/destinationqueue.go index 12e6db9fac..a4542c4985 100644 --- a/federationapi/queue/destinationqueue.go +++ b/federationapi/queue/destinationqueue.go @@ -23,6 +23,7 @@ import ( "github.com/matrix-org/gomatrix" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/sirupsen/logrus" "go.uber.org/atomic" @@ -50,7 +51,7 @@ type destinationQueue struct { queues *OutgoingQueues db storage.Database process *process.ProcessContext - signing map[gomatrixserverlib.ServerName]*gomatrixserverlib.SigningIdentity + signing map[gomatrixserverlib.ServerName]*fclient.SigningIdentity rsAPI api.FederationRoomserverAPI client fedapi.FederationClient // federation client origin gomatrixserverlib.ServerName // origin of requests diff --git a/federationapi/queue/queue.go b/federationapi/queue/queue.go index 5d6b8d44c8..c0ecb28759 100644 --- a/federationapi/queue/queue.go +++ b/federationapi/queue/queue.go @@ -22,6 +22,7 @@ import ( "github.com/getsentry/sentry-go" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/prometheus/client_golang/prometheus" "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus" @@ -45,7 +46,7 @@ type OutgoingQueues struct { origin gomatrixserverlib.ServerName client fedapi.FederationClient statistics *statistics.Statistics - signing map[gomatrixserverlib.ServerName]*gomatrixserverlib.SigningIdentity + signing map[gomatrixserverlib.ServerName]*fclient.SigningIdentity queuesMutex sync.Mutex // protects the below queues map[gomatrixserverlib.ServerName]*destinationQueue } @@ -90,7 +91,7 @@ func NewOutgoingQueues( client fedapi.FederationClient, rsAPI api.FederationRoomserverAPI, statistics *statistics.Statistics, - signing []*gomatrixserverlib.SigningIdentity, + signing []*fclient.SigningIdentity, ) *OutgoingQueues { queues := &OutgoingQueues{ disabled: disabled, @@ -100,7 +101,7 @@ func NewOutgoingQueues( origin: origin, client: client, statistics: statistics, - signing: map[gomatrixserverlib.ServerName]*gomatrixserverlib.SigningIdentity{}, + signing: map[gomatrixserverlib.ServerName]*fclient.SigningIdentity{}, queues: map[gomatrixserverlib.ServerName]*destinationQueue{}, } for _, identity := range signing { diff --git a/federationapi/queue/queue_test.go b/federationapi/queue/queue_test.go index 55d1df4a24..65a925d348 100644 --- a/federationapi/queue/queue_test.go +++ b/federationapi/queue/queue_test.go @@ -24,6 +24,7 @@ import ( "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/test/testrig" + "github.com/matrix-org/gomatrixserverlib/fclient" "go.uber.org/atomic" "gotest.tools/v3/poll" @@ -80,24 +81,24 @@ type stubFederationClient struct { txRelayCount atomic.Uint32 } -func (f *stubFederationClient) SendTransaction(ctx context.Context, t gomatrixserverlib.Transaction) (res gomatrixserverlib.RespSend, err error) { +func (f *stubFederationClient) SendTransaction(ctx context.Context, t gomatrixserverlib.Transaction) (res fclient.RespSend, err error) { var result error if !f.shouldTxSucceed { result = fmt.Errorf("transaction failed") } f.txCount.Add(1) - return gomatrixserverlib.RespSend{}, result + return fclient.RespSend{}, result } -func (f *stubFederationClient) P2PSendTransactionToRelay(ctx context.Context, u gomatrixserverlib.UserID, t gomatrixserverlib.Transaction, forwardingServer gomatrixserverlib.ServerName) (res gomatrixserverlib.EmptyResp, err error) { +func (f *stubFederationClient) P2PSendTransactionToRelay(ctx context.Context, u gomatrixserverlib.UserID, t gomatrixserverlib.Transaction, forwardingServer gomatrixserverlib.ServerName) (res fclient.EmptyResp, err error) { var result error if !f.shouldTxRelaySucceed { result = fmt.Errorf("relay transaction failed") } f.txRelayCount.Add(1) - return gomatrixserverlib.EmptyResp{}, result + return fclient.EmptyResp{}, result } func mustCreatePDU(t *testing.T) *gomatrixserverlib.HeaderedEvent { @@ -127,7 +128,7 @@ func testSetup(failuresUntilBlacklist uint32, failuresUntilAssumedOffline uint32 rs := &stubFederationRoomServerAPI{} stats := statistics.NewStatistics(db, failuresUntilBlacklist, failuresUntilAssumedOffline) - signingInfo := []*gomatrixserverlib.SigningIdentity{ + signingInfo := []*fclient.SigningIdentity{ { KeyID: "ed21019:auto", PrivateKey: test.PrivateKeyA, diff --git a/federationapi/routing/devices.go b/federationapi/routing/devices.go index 871d26cd4b..aae1299fe1 100644 --- a/federationapi/routing/devices.go +++ b/federationapi/routing/devices.go @@ -19,6 +19,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/matrix-org/util" "github.com/tidwall/gjson" ) @@ -53,10 +54,10 @@ func GetUserDevices( return jsonerror.InternalAPIError(req.Context(), err) } - response := gomatrixserverlib.RespUserDevices{ + response := fclient.RespUserDevices{ UserID: userID, StreamID: res.StreamID, - Devices: []gomatrixserverlib.RespUserDevice{}, + Devices: []fclient.RespUserDevice{}, } if masterKey, ok := sigRes.MasterKeys[userID]; ok { @@ -67,7 +68,7 @@ func GetUserDevices( } for _, dev := range res.Devices { - var key gomatrixserverlib.RespUserDeviceKeys + var key fclient.RespUserDeviceKeys err := json.Unmarshal(dev.DeviceKeys.KeyJSON, &key) if err != nil { util.GetLogger(req.Context()).WithError(err).Warnf("malformed device key: %s", string(dev.DeviceKeys.KeyJSON)) @@ -79,7 +80,7 @@ func GetUserDevices( displayName = gjson.GetBytes(dev.DeviceKeys.KeyJSON, "unsigned.device_display_name").Str } - device := gomatrixserverlib.RespUserDevice{ + device := fclient.RespUserDevice{ DeviceID: dev.DeviceID, DisplayName: displayName, Keys: key, diff --git a/federationapi/routing/eventauth.go b/federationapi/routing/eventauth.go index 2f1f3baf66..65a2a9bc8a 100644 --- a/federationapi/routing/eventauth.go +++ b/federationapi/routing/eventauth.go @@ -19,6 +19,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/matrix-org/util" ) @@ -70,7 +71,7 @@ func GetEventAuth( return util.JSONResponse{ Code: http.StatusOK, - JSON: gomatrixserverlib.RespEventAuth{ + JSON: fclient.RespEventAuth{ AuthEvents: gomatrixserverlib.NewEventJSONsFromHeaderedEvents(response.AuthChainEvents), }, } diff --git a/federationapi/routing/invite.go b/federationapi/routing/invite.go index f424fcacd9..c1fdf266b3 100644 --- a/federationapi/routing/invite.go +++ b/federationapi/routing/invite.go @@ -25,6 +25,7 @@ import ( roomserverVersion "github.com/matrix-org/dendrite/roomserver/version" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/matrix-org/util" ) @@ -218,12 +219,12 @@ func processInvite( if isInviteV2 { return util.JSONResponse{ Code: http.StatusOK, - JSON: gomatrixserverlib.RespInviteV2{Event: signedEvent.JSON()}, + JSON: fclient.RespInviteV2{Event: signedEvent.JSON()}, } } else { return util.JSONResponse{ Code: http.StatusOK, - JSON: gomatrixserverlib.RespInvite{Event: signedEvent.JSON()}, + JSON: fclient.RespInvite{Event: signedEvent.JSON()}, } } } diff --git a/federationapi/routing/join.go b/federationapi/routing/join.go index 03809df751..1476f903f8 100644 --- a/federationapi/routing/join.go +++ b/federationapi/routing/join.go @@ -22,6 +22,7 @@ import ( "time" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/matrix-org/util" "github.com/sirupsen/logrus" @@ -433,7 +434,7 @@ func SendJoin( // https://matrix.org/docs/spec/server_server/latest#put-matrix-federation-v1-send-join-roomid-eventid return util.JSONResponse{ Code: http.StatusOK, - JSON: gomatrixserverlib.RespSendJoin{ + JSON: fclient.RespSendJoin{ StateEvents: gomatrixserverlib.NewEventJSONsFromHeaderedEvents(stateAndAuthChainResponse.StateEvents), AuthEvents: gomatrixserverlib.NewEventJSONsFromHeaderedEvents(stateAndAuthChainResponse.AuthChainEvents), Origin: cfg.Matrix.ServerName, diff --git a/federationapi/routing/keys.go b/federationapi/routing/keys.go index 2885cc916d..db768591f7 100644 --- a/federationapi/routing/keys.go +++ b/federationapi/routing/keys.go @@ -25,6 +25,7 @@ import ( "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/matrix-org/util" "github.com/sirupsen/logrus" "golang.org/x/crypto/ed25519" @@ -144,7 +145,7 @@ func LocalKeys(cfg *config.FederationAPI, serverName gomatrixserverlib.ServerNam func localKeys(cfg *config.FederationAPI, serverName gomatrixserverlib.ServerName) (*gomatrixserverlib.ServerKeys, error) { var keys gomatrixserverlib.ServerKeys - var identity *gomatrixserverlib.SigningIdentity + var identity *fclient.SigningIdentity var err error if virtualHost := cfg.Matrix.VirtualHostForHTTPHost(serverName); virtualHost == nil { if identity, err = cfg.Matrix.SigningIdentityFor(cfg.Matrix.ServerName); err != nil { diff --git a/federationapi/routing/missingevents.go b/federationapi/routing/missingevents.go index 531cb9e28c..63a32b9c45 100644 --- a/federationapi/routing/missingevents.go +++ b/federationapi/routing/missingevents.go @@ -19,6 +19,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/matrix-org/util" ) @@ -67,7 +68,7 @@ func GetMissingEvents( eventsResponse.Events = filterEvents(eventsResponse.Events, roomID) - resp := gomatrixserverlib.RespMissingEvents{ + resp := fclient.RespMissingEvents{ Events: gomatrixserverlib.NewEventJSONsFromHeaderedEvents(eventsResponse.Events), } diff --git a/federationapi/routing/peek.go b/federationapi/routing/peek.go index bc4dac90f9..6c4d315c0c 100644 --- a/federationapi/routing/peek.go +++ b/federationapi/routing/peek.go @@ -21,6 +21,7 @@ import ( "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/matrix-org/util" ) @@ -87,7 +88,7 @@ func Peek( return util.JSONResponse{Code: http.StatusNotFound, JSON: nil} } - respPeek := gomatrixserverlib.RespPeek{ + respPeek := fclient.RespPeek{ StateEvents: gomatrixserverlib.NewEventJSONsFromHeaderedEvents(response.StateEvents), AuthEvents: gomatrixserverlib.NewEventJSONsFromHeaderedEvents(response.AuthChainEvents), RoomVersion: response.RoomVersion, diff --git a/federationapi/routing/publicrooms.go b/federationapi/routing/publicrooms.go index 34025932a8..7c5d6a02e1 100644 --- a/federationapi/routing/publicrooms.go +++ b/federationapi/routing/publicrooms.go @@ -7,6 +7,7 @@ import ( "strconv" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/matrix-org/util" "github.com/matrix-org/dendrite/clientapi/httputil" @@ -48,9 +49,9 @@ func GetPostPublicRooms(req *http.Request, rsAPI roomserverAPI.FederationRoomser func publicRooms( ctx context.Context, request PublicRoomReq, rsAPI roomserverAPI.FederationRoomserverAPI, -) (*gomatrixserverlib.RespPublicRooms, error) { +) (*fclient.RespPublicRooms, error) { - var response gomatrixserverlib.RespPublicRooms + var response fclient.RespPublicRooms var limit int16 var offset int64 limit = request.Limit @@ -122,7 +123,7 @@ func fillPublicRoomsReq(httpReq *http.Request, request *PublicRoomReq) *util.JSO } // due to lots of switches -func fillInRooms(ctx context.Context, roomIDs []string, rsAPI roomserverAPI.FederationRoomserverAPI) ([]gomatrixserverlib.PublicRoom, error) { +func fillInRooms(ctx context.Context, roomIDs []string, rsAPI roomserverAPI.FederationRoomserverAPI) ([]fclient.PublicRoom, error) { avatarTuple := gomatrixserverlib.StateKeyTuple{EventType: "m.room.avatar", StateKey: ""} nameTuple := gomatrixserverlib.StateKeyTuple{EventType: "m.room.name", StateKey: ""} canonicalTuple := gomatrixserverlib.StateKeyTuple{EventType: gomatrixserverlib.MRoomCanonicalAlias, StateKey: ""} @@ -144,10 +145,10 @@ func fillInRooms(ctx context.Context, roomIDs []string, rsAPI roomserverAPI.Fede util.GetLogger(ctx).WithError(err).Error("QueryBulkStateContent failed") return nil, err } - chunk := make([]gomatrixserverlib.PublicRoom, len(roomIDs)) + chunk := make([]fclient.PublicRoom, len(roomIDs)) i := 0 for roomID, data := range stateRes.Rooms { - pub := gomatrixserverlib.PublicRoom{ + pub := fclient.PublicRoom{ RoomID: roomID, } joinCount := 0 diff --git a/federationapi/routing/query.go b/federationapi/routing/query.go index e6dc52601a..6b1c371ecd 100644 --- a/federationapi/routing/query.go +++ b/federationapi/routing/query.go @@ -24,6 +24,7 @@ import ( "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/gomatrix" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/matrix-org/util" ) @@ -50,7 +51,7 @@ func RoomAliasToID( } } - var resp gomatrixserverlib.RespDirectory + var resp fclient.RespDirectory if domain == cfg.Matrix.ServerName { queryReq := &roomserverAPI.GetRoomIDForAliasRequest{ @@ -71,7 +72,7 @@ func RoomAliasToID( return jsonerror.InternalServerError() } - resp = gomatrixserverlib.RespDirectory{ + resp = fclient.RespDirectory{ RoomID: queryRes.RoomID, Servers: serverQueryRes.ServerNames, } diff --git a/federationapi/routing/query_test.go b/federationapi/routing/query_test.go index 6e702963f6..807e7b2f2d 100644 --- a/federationapi/routing/query_test.go +++ b/federationapi/routing/query_test.go @@ -35,6 +35,7 @@ import ( "github.com/matrix-org/dendrite/test" "github.com/matrix-org/dendrite/test/testrig" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/stretchr/testify/assert" "golang.org/x/crypto/ed25519" ) @@ -43,7 +44,7 @@ type fakeFedClient struct { fedclient.FederationClient } -func (f *fakeFedClient) LookupRoomAlias(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomAlias string) (res gomatrixserverlib.RespDirectory, err error) { +func (f *fakeFedClient) LookupRoomAlias(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomAlias string) (res fclient.RespDirectory, err error) { return } diff --git a/federationapi/routing/state.go b/federationapi/routing/state.go index 1120cf260e..1152c09323 100644 --- a/federationapi/routing/state.go +++ b/federationapi/routing/state.go @@ -20,6 +20,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/matrix-org/util" ) @@ -40,7 +41,7 @@ func GetState( return *err } - return util.JSONResponse{Code: http.StatusOK, JSON: &gomatrixserverlib.RespState{ + return util.JSONResponse{Code: http.StatusOK, JSON: &fclient.RespState{ AuthEvents: gomatrixserverlib.NewEventJSONsFromHeaderedEvents(authChain), StateEvents: gomatrixserverlib.NewEventJSONsFromHeaderedEvents(stateEvents), }} @@ -66,7 +67,7 @@ func GetStateIDs( stateEventIDs := getIDsFromEvent(stateEvents) authEventIDs := getIDsFromEvent(authEvents) - return util.JSONResponse{Code: http.StatusOK, JSON: gomatrixserverlib.RespStateIDs{ + return util.JSONResponse{Code: http.StatusOK, JSON: fclient.RespStateIDs{ StateEventIDs: stateEventIDs, AuthEventIDs: authEventIDs, }, diff --git a/go.mod b/go.mod index 7ac294f2a1..fc185dd182 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530 - github.com/matrix-org/gomatrixserverlib v0.0.0-20230320105331-4dd7ff2f0e3a + github.com/matrix-org/gomatrixserverlib v0.0.0-20230405171344-5f597d85ba4f github.com/matrix-org/pinecone v0.11.1-0.20230210171230-8c3b24f2649a github.com/matrix-org/util v0.0.0-20221111132719-399730281e66 github.com/mattn/go-sqlite3 v1.14.16 diff --git a/go.sum b/go.sum index bf8fbdf3db..4ee6d5e31d 100644 --- a/go.sum +++ b/go.sum @@ -323,6 +323,8 @@ github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530 h1:kHKxCOLcHH8 github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= github.com/matrix-org/gomatrixserverlib v0.0.0-20230320105331-4dd7ff2f0e3a h1:F6K1i61KcJ8cX/y0Q8/44Dh1w+fpESQd92gq885FDrI= github.com/matrix-org/gomatrixserverlib v0.0.0-20230320105331-4dd7ff2f0e3a/go.mod h1:7HTbSZe+CIdmeqVyFMekwD5dFU8khWQyngKATvd12FU= +github.com/matrix-org/gomatrixserverlib v0.0.0-20230405171344-5f597d85ba4f h1:D7IgZA2DxBroqCTxo2uXEmjj8eCI1OzqqKRE4SAgmBU= +github.com/matrix-org/gomatrixserverlib v0.0.0-20230405171344-5f597d85ba4f/go.mod h1:7HTbSZe+CIdmeqVyFMekwD5dFU8khWQyngKATvd12FU= github.com/matrix-org/pinecone v0.11.1-0.20230210171230-8c3b24f2649a h1:awrPDf9LEFySxTLKYBMCiObelNx/cBuv/wzllvCCH3A= github.com/matrix-org/pinecone v0.11.1-0.20230210171230-8c3b24f2649a/go.mod h1:HchJX9oKMXaT2xYFs0Ha/6Zs06mxLU8k6F1ODnrGkeQ= github.com/matrix-org/util v0.0.0-20221111132719-399730281e66 h1:6z4KxomXSIGWqhHcfzExgkH3Z3UkIXry4ibJS4Aqz2Y= diff --git a/internal/caching/cache_space_rooms.go b/internal/caching/cache_space_rooms.go index 697f992695..100ab9023e 100644 --- a/internal/caching/cache_space_rooms.go +++ b/internal/caching/cache_space_rooms.go @@ -1,18 +1,16 @@ package caching -import ( - "github.com/matrix-org/gomatrixserverlib" -) +import "github.com/matrix-org/gomatrixserverlib/fclient" type SpaceSummaryRoomsCache interface { - GetSpaceSummary(roomID string) (r gomatrixserverlib.MSC2946SpacesResponse, ok bool) - StoreSpaceSummary(roomID string, r gomatrixserverlib.MSC2946SpacesResponse) + GetSpaceSummary(roomID string) (r fclient.MSC2946SpacesResponse, ok bool) + StoreSpaceSummary(roomID string, r fclient.MSC2946SpacesResponse) } -func (c Caches) GetSpaceSummary(roomID string) (r gomatrixserverlib.MSC2946SpacesResponse, ok bool) { +func (c Caches) GetSpaceSummary(roomID string) (r fclient.MSC2946SpacesResponse, ok bool) { return c.SpaceSummaryRooms.Get(roomID) } -func (c Caches) StoreSpaceSummary(roomID string, r gomatrixserverlib.MSC2946SpacesResponse) { +func (c Caches) StoreSpaceSummary(roomID string, r fclient.MSC2946SpacesResponse) { c.SpaceSummaryRooms.Set(roomID, r) } diff --git a/internal/caching/caches.go b/internal/caching/caches.go index 479920466d..a678632ebe 100644 --- a/internal/caching/caches.go +++ b/internal/caching/caches.go @@ -17,6 +17,7 @@ package caching import ( "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/fclient" ) // Caches contains a set of references to caches. They may be @@ -34,7 +35,7 @@ type Caches struct { RoomServerEventTypes Cache[types.EventTypeNID, string] // eventType NID -> eventType FederationPDUs Cache[int64, *gomatrixserverlib.HeaderedEvent] // queue NID -> PDU FederationEDUs Cache[int64, *gomatrixserverlib.EDU] // queue NID -> EDU - SpaceSummaryRooms Cache[string, gomatrixserverlib.MSC2946SpacesResponse] // room ID -> space response + SpaceSummaryRooms Cache[string, fclient.MSC2946SpacesResponse] // room ID -> space response LazyLoading Cache[lazyLoadingCacheKey, string] // composite key -> event ID } diff --git a/internal/caching/impl_ristretto.go b/internal/caching/impl_ristretto.go index 7663ddcb95..4656b6b7eb 100644 --- a/internal/caching/impl_ristretto.go +++ b/internal/caching/impl_ristretto.go @@ -23,6 +23,7 @@ import ( "github.com/dgraph-io/ristretto" "github.com/dgraph-io/ristretto/z" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" @@ -146,7 +147,7 @@ func NewRistrettoCache(maxCost config.DataUnit, maxAge time.Duration, enableProm MaxAge: lesserOf(time.Hour/2, maxAge), }, }, - SpaceSummaryRooms: &RistrettoCachePartition[string, gomatrixserverlib.MSC2946SpacesResponse]{ // room ID -> space response + SpaceSummaryRooms: &RistrettoCachePartition[string, fclient.MSC2946SpacesResponse]{ // room ID -> space response cache: cache, Prefix: spaceSummaryRoomsCache, Mutable: true, diff --git a/internal/eventutil/events.go b/internal/eventutil/events.go index c572d8830e..984a3f5397 100644 --- a/internal/eventutil/events.go +++ b/internal/eventutil/events.go @@ -22,6 +22,7 @@ import ( "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/matrix-org/gomatrixserverlib" ) @@ -39,7 +40,7 @@ var ErrRoomNoExists = errors.New("room does not exist") func QueryAndBuildEvent( ctx context.Context, builder *gomatrixserverlib.EventBuilder, cfg *config.Global, - identity *gomatrixserverlib.SigningIdentity, evTime time.Time, + identity *fclient.SigningIdentity, evTime time.Time, rsAPI api.QueryLatestEventsAndStateAPI, queryRes *api.QueryLatestEventsAndStateResponse, ) (*gomatrixserverlib.HeaderedEvent, error) { if queryRes == nil { @@ -59,7 +60,7 @@ func QueryAndBuildEvent( func BuildEvent( ctx context.Context, builder *gomatrixserverlib.EventBuilder, cfg *config.Global, - identity *gomatrixserverlib.SigningIdentity, evTime time.Time, + identity *fclient.SigningIdentity, evTime time.Time, eventsNeeded *gomatrixserverlib.StateNeeded, queryRes *api.QueryLatestEventsAndStateResponse, ) (*gomatrixserverlib.HeaderedEvent, error) { if err := addPrevEventsToEvent(builder, eventsNeeded, queryRes); err != nil { diff --git a/internal/transactionrequest.go b/internal/transactionrequest.go index 13b00af505..60c02b1297 100644 --- a/internal/transactionrequest.go +++ b/internal/transactionrequest.go @@ -28,6 +28,7 @@ import ( syncTypes "github.com/matrix-org/dendrite/syncapi/types" userAPI "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/matrix-org/util" "github.com/prometheus/client_golang/prometheus" "github.com/sirupsen/logrus" @@ -97,7 +98,7 @@ func NewTxnReq( return t } -func (t *TxnReq) ProcessTransaction(ctx context.Context) (*gomatrixserverlib.RespSend, *util.JSONResponse) { +func (t *TxnReq) ProcessTransaction(ctx context.Context) (*fclient.RespSend, *util.JSONResponse) { var wg sync.WaitGroup wg.Add(1) go func() { @@ -107,7 +108,7 @@ func (t *TxnReq) ProcessTransaction(ctx context.Context) (*gomatrixserverlib.Res } }() - results := make(map[string]gomatrixserverlib.PDUResult) + results := make(map[string]fclient.PDUResult) roomVersions := make(map[string]gomatrixserverlib.RoomVersion) getRoomVersion := func(roomID string) gomatrixserverlib.RoomVersion { if v, ok := roomVersions[roomID]; ok { @@ -157,14 +158,14 @@ func (t *TxnReq) ProcessTransaction(ctx context.Context) (*gomatrixserverlib.Res continue } if api.IsServerBannedFromRoom(ctx, t.rsAPI, event.RoomID(), t.Origin) { - results[event.EventID()] = gomatrixserverlib.PDUResult{ + results[event.EventID()] = fclient.PDUResult{ Error: "Forbidden by server ACLs", } continue } if err = event.VerifyEventSignatures(ctx, t.keys); err != nil { util.GetLogger(ctx).WithError(err).Debugf("Transaction: Couldn't validate signature of event %q", event.EventID()) - results[event.EventID()] = gomatrixserverlib.PDUResult{ + results[event.EventID()] = fclient.PDUResult{ Error: err.Error(), } continue @@ -187,18 +188,18 @@ func (t *TxnReq) ProcessTransaction(ctx context.Context) (*gomatrixserverlib.Res true, ); err != nil { util.GetLogger(ctx).WithError(err).Errorf("Transaction: Couldn't submit event %q to input queue: %s", event.EventID(), err) - results[event.EventID()] = gomatrixserverlib.PDUResult{ + results[event.EventID()] = fclient.PDUResult{ Error: err.Error(), } continue } - results[event.EventID()] = gomatrixserverlib.PDUResult{} + results[event.EventID()] = fclient.PDUResult{} PDUCountTotal.WithLabelValues("success").Inc() } wg.Wait() - return &gomatrixserverlib.RespSend{PDUs: results}, nil + return &fclient.RespSend{PDUs: results}, nil } // nolint:gocyclo diff --git a/mediaapi/mediaapi.go b/mediaapi/mediaapi.go index 5d517ef228..284071a536 100644 --- a/mediaapi/mediaapi.go +++ b/mediaapi/mediaapi.go @@ -21,7 +21,7 @@ import ( "github.com/matrix-org/dendrite/mediaapi/storage" "github.com/matrix-org/dendrite/setup/config" userapi "github.com/matrix-org/dendrite/userapi/api" - "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/sirupsen/logrus" ) @@ -31,7 +31,7 @@ func AddPublicRoutes( cm sqlutil.Connections, cfg *config.Dendrite, userAPI userapi.MediaUserAPI, - client *gomatrixserverlib.Client, + client *fclient.Client, ) { mediaDB, err := storage.NewMediaAPIDatasource(cm, &cfg.MediaAPI.Database) if err != nil { diff --git a/mediaapi/routing/download.go b/mediaapi/routing/download.go index c9299b1fc1..412faceb30 100644 --- a/mediaapi/routing/download.go +++ b/mediaapi/routing/download.go @@ -37,6 +37,7 @@ import ( "github.com/matrix-org/dendrite/mediaapi/types" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/matrix-org/util" "github.com/pkg/errors" log "github.com/sirupsen/logrus" @@ -75,7 +76,7 @@ func Download( mediaID types.MediaID, cfg *config.MediaAPI, db storage.Database, - client *gomatrixserverlib.Client, + client *fclient.Client, activeRemoteRequests *types.ActiveRemoteRequests, activeThumbnailGeneration *types.ActiveThumbnailGeneration, isThumbnailRequest bool, @@ -205,7 +206,7 @@ func (r *downloadRequest) doDownload( w http.ResponseWriter, cfg *config.MediaAPI, db storage.Database, - client *gomatrixserverlib.Client, + client *fclient.Client, activeRemoteRequests *types.ActiveRemoteRequests, activeThumbnailGeneration *types.ActiveThumbnailGeneration, ) (*types.MediaMetadata, error) { @@ -513,7 +514,7 @@ func (r *downloadRequest) generateThumbnail( // Note: The named errorResponse return variable is used in a deferred broadcast of the metadata and error response to waiting goroutines. func (r *downloadRequest) getRemoteFile( ctx context.Context, - client *gomatrixserverlib.Client, + client *fclient.Client, cfg *config.MediaAPI, db storage.Database, activeRemoteRequests *types.ActiveRemoteRequests, @@ -615,7 +616,7 @@ func (r *downloadRequest) broadcastMediaMetadata(activeRemoteRequests *types.Act // fetchRemoteFileAndStoreMetadata fetches the file from the remote server and stores its metadata in the database func (r *downloadRequest) fetchRemoteFileAndStoreMetadata( ctx context.Context, - client *gomatrixserverlib.Client, + client *fclient.Client, absBasePath config.Path, maxFileSizeBytes config.FileSizeBytes, db storage.Database, @@ -713,7 +714,7 @@ func (r *downloadRequest) GetContentLengthAndReader(contentLengthHeader string, func (r *downloadRequest) fetchRemoteFile( ctx context.Context, - client *gomatrixserverlib.Client, + client *fclient.Client, absBasePath config.Path, maxFileSizeBytes config.FileSizeBytes, ) (types.Path, bool, error) { diff --git a/mediaapi/routing/routing.go b/mediaapi/routing/routing.go index 98e6a82edd..79e8308ae8 100644 --- a/mediaapi/routing/routing.go +++ b/mediaapi/routing/routing.go @@ -26,6 +26,7 @@ import ( "github.com/matrix-org/dendrite/setup/config" userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/matrix-org/util" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" @@ -48,7 +49,7 @@ func Setup( cfg *config.Dendrite, db storage.Database, userAPI userapi.MediaUserAPI, - client *gomatrixserverlib.Client, + client *fclient.Client, ) { rateLimits := httputil.NewRateLimits(&cfg.ClientAPI.RateLimiting) @@ -103,7 +104,7 @@ func makeDownloadAPI( cfg *config.MediaAPI, rateLimits *httputil.RateLimits, db storage.Database, - client *gomatrixserverlib.Client, + client *fclient.Client, activeRemoteRequests *types.ActiveRemoteRequests, activeThumbnailGeneration *types.ActiveThumbnailGeneration, ) http.HandlerFunc { diff --git a/relayapi/api/api.go b/relayapi/api/api.go index 9b4b62e58d..f0ed83262b 100644 --- a/relayapi/api/api.go +++ b/relayapi/api/api.go @@ -18,6 +18,7 @@ import ( "context" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/fclient" ) // RelayInternalAPI is used to query information from the relay server. @@ -51,7 +52,7 @@ type RelayServerAPI interface { QueryTransactions( ctx context.Context, userID gomatrixserverlib.UserID, - previousEntry gomatrixserverlib.RelayEntry, + previousEntry fclient.RelayEntry, ) (QueryRelayTransactionsResponse, error) } diff --git a/relayapi/internal/perform.go b/relayapi/internal/perform.go index 62c7d446e4..66d421190f 100644 --- a/relayapi/internal/perform.go +++ b/relayapi/internal/perform.go @@ -21,6 +21,7 @@ import ( "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/relayapi/api" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/sirupsen/logrus" ) @@ -46,7 +47,7 @@ func (r *RelayInternalAPI) PerformRelayServerSync( ) error { // Providing a default RelayEntry (EntryID = 0) is done to ask the relay if there are any // transactions available for this node. - prevEntry := gomatrixserverlib.RelayEntry{} + prevEntry := fclient.RelayEntry{} asyncResponse, err := r.fedClient.P2PGetTransactionFromRelay(ctx, userID, prevEntry, relayServer) if err != nil { logrus.Errorf("P2PGetTransactionFromRelay: %s", err.Error()) @@ -54,12 +55,12 @@ func (r *RelayInternalAPI) PerformRelayServerSync( } r.processTransaction(&asyncResponse.Transaction) - prevEntry = gomatrixserverlib.RelayEntry{EntryID: asyncResponse.EntryID} + prevEntry = fclient.RelayEntry{EntryID: asyncResponse.EntryID} for asyncResponse.EntriesQueued { // There are still more entries available for this node from the relay. logrus.Infof("Retrieving next entry from relay, previous: %v", prevEntry) asyncResponse, err = r.fedClient.P2PGetTransactionFromRelay(ctx, userID, prevEntry, relayServer) - prevEntry = gomatrixserverlib.RelayEntry{EntryID: asyncResponse.EntryID} + prevEntry = fclient.RelayEntry{EntryID: asyncResponse.EntryID} if err != nil { logrus.Errorf("P2PGetTransactionFromRelay: %s", err.Error()) return err @@ -97,7 +98,7 @@ func (r *RelayInternalAPI) PerformStoreTransaction( func (r *RelayInternalAPI) QueryTransactions( ctx context.Context, userID gomatrixserverlib.UserID, - previousEntry gomatrixserverlib.RelayEntry, + previousEntry fclient.RelayEntry, ) (api.QueryRelayTransactionsResponse, error) { logrus.Infof("QueryTransactions for %s", userID.Raw()) if previousEntry.EntryID > 0 { diff --git a/relayapi/internal/perform_test.go b/relayapi/internal/perform_test.go index 278706a3e4..2c5e1f1f0c 100644 --- a/relayapi/internal/perform_test.go +++ b/relayapi/internal/perform_test.go @@ -24,6 +24,7 @@ import ( "github.com/matrix-org/dendrite/relayapi/storage/shared" "github.com/matrix-org/dendrite/test" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/stretchr/testify/assert" ) @@ -37,15 +38,15 @@ type testFedClient struct { func (f *testFedClient) P2PGetTransactionFromRelay( ctx context.Context, u gomatrixserverlib.UserID, - prev gomatrixserverlib.RelayEntry, + prev fclient.RelayEntry, relayServer gomatrixserverlib.ServerName, -) (res gomatrixserverlib.RespGetRelayTransaction, err error) { +) (res fclient.RespGetRelayTransaction, err error) { f.queryCount++ if f.shouldFail { return res, fmt.Errorf("Error") } - res = gomatrixserverlib.RespGetRelayTransaction{ + res = fclient.RespGetRelayTransaction{ Transaction: gomatrixserverlib.Transaction{}, EntryID: 0, } diff --git a/relayapi/relayapi.go b/relayapi/relayapi.go index fdc366e4c3..bae6b0cf55 100644 --- a/relayapi/relayapi.go +++ b/relayapi/relayapi.go @@ -26,6 +26,7 @@ import ( rsAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/sirupsen/logrus" ) @@ -53,7 +54,7 @@ func AddPublicRoutes( func NewRelayInternalAPI( dendriteCfg *config.Dendrite, cm sqlutil.Connections, - fedClient *gomatrixserverlib.FederationClient, + fedClient *fclient.FederationClient, rsAPI rsAPI.RoomserverInternalAPI, keyRing *gomatrixserverlib.KeyRing, producer *producers.SyncAPIProducer, diff --git a/relayapi/relayapi_test.go b/relayapi/relayapi_test.go index 8973c8cc94..27426221c1 100644 --- a/relayapi/relayapi_test.go +++ b/relayapi/relayapi_test.go @@ -32,6 +32,7 @@ import ( "github.com/matrix-org/dendrite/test" "github.com/matrix-org/dendrite/test/testrig" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/stretchr/testify/assert" ) @@ -79,7 +80,7 @@ func createGetRelayTxnHTTPRequest(serverName gomatrixserverlib.ServerName, userI pk := sk.Public().(ed25519.PublicKey) origin := gomatrixserverlib.ServerName(hex.EncodeToString(pk)) req := gomatrixserverlib.NewFederationRequest("GET", origin, serverName, "/_matrix/federation/v1/relay_txn/"+userID) - content := gomatrixserverlib.RelayEntry{EntryID: 0} + content := fclient.RelayEntry{EntryID: 0} req.SetContent(content) req.Sign(origin, gomatrixserverlib.KeyID(keyID), sk) httpreq, _ := req.HTTPRequest() diff --git a/relayapi/routing/relaytxn.go b/relayapi/routing/relaytxn.go index 63b42ec7dc..06a2a07f2b 100644 --- a/relayapi/routing/relaytxn.go +++ b/relayapi/routing/relaytxn.go @@ -21,6 +21,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/relayapi/api" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/matrix-org/util" "github.com/sirupsen/logrus" ) @@ -35,7 +36,7 @@ func GetTransactionFromRelay( ) util.JSONResponse { logrus.Infof("Processing relay_txn for %s", userID.Raw()) - var previousEntry gomatrixserverlib.RelayEntry + var previousEntry fclient.RelayEntry if err := json.Unmarshal(fedReq.Content(), &previousEntry); err != nil { return util.JSONResponse{ Code: http.StatusInternalServerError, @@ -59,7 +60,7 @@ func GetTransactionFromRelay( return util.JSONResponse{ Code: http.StatusOK, - JSON: gomatrixserverlib.RespGetRelayTransaction{ + JSON: fclient.RespGetRelayTransaction{ Transaction: response.Transaction, EntryID: response.EntryID, EntriesQueued: response.EntriesQueued, diff --git a/relayapi/routing/relaytxn_test.go b/relayapi/routing/relaytxn_test.go index 4c099a642d..bc76ddf2c0 100644 --- a/relayapi/routing/relaytxn_test.go +++ b/relayapi/routing/relaytxn_test.go @@ -25,12 +25,13 @@ import ( "github.com/matrix-org/dendrite/relayapi/storage/shared" "github.com/matrix-org/dendrite/test" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/stretchr/testify/assert" ) func createQuery( userID gomatrixserverlib.UserID, - prevEntry gomatrixserverlib.RelayEntry, + prevEntry fclient.RelayEntry, ) gomatrixserverlib.FederationRequest { var federationPathPrefixV1 = "/_matrix/federation/v1" path := federationPathPrefixV1 + "/relay_txn/" + userID.Raw() @@ -60,11 +61,11 @@ func TestGetEmptyDatabaseReturnsNothing(t *testing.T) { &db, nil, nil, nil, nil, false, "", true, ) - request := createQuery(*userID, gomatrixserverlib.RelayEntry{}) + request := createQuery(*userID, fclient.RelayEntry{}) response := routing.GetTransactionFromRelay(httpReq, &request, relayAPI, *userID) assert.Equal(t, http.StatusOK, response.Code) - jsonResponse := response.JSON.(gomatrixserverlib.RespGetRelayTransaction) + jsonResponse := response.JSON.(fclient.RespGetRelayTransaction) assert.Equal(t, false, jsonResponse.EntriesQueued) assert.Equal(t, gomatrixserverlib.Transaction{}, jsonResponse.Transaction) @@ -93,7 +94,7 @@ func TestGetInvalidPrevEntryFails(t *testing.T) { &db, nil, nil, nil, nil, false, "", true, ) - request := createQuery(*userID, gomatrixserverlib.RelayEntry{EntryID: -1}) + request := createQuery(*userID, fclient.RelayEntry{EntryID: -1}) response := routing.GetTransactionFromRelay(httpReq, &request, relayAPI, *userID) assert.Equal(t, http.StatusInternalServerError, response.Code) } @@ -126,20 +127,20 @@ func TestGetReturnsSavedTransaction(t *testing.T) { &db, nil, nil, nil, nil, false, "", true, ) - request := createQuery(*userID, gomatrixserverlib.RelayEntry{}) + request := createQuery(*userID, fclient.RelayEntry{}) response := routing.GetTransactionFromRelay(httpReq, &request, relayAPI, *userID) assert.Equal(t, http.StatusOK, response.Code) - jsonResponse := response.JSON.(gomatrixserverlib.RespGetRelayTransaction) + jsonResponse := response.JSON.(fclient.RespGetRelayTransaction) assert.True(t, jsonResponse.EntriesQueued) assert.Equal(t, transaction, jsonResponse.Transaction) // And once more to clear the queue - request = createQuery(*userID, gomatrixserverlib.RelayEntry{EntryID: jsonResponse.EntryID}) + request = createQuery(*userID, fclient.RelayEntry{EntryID: jsonResponse.EntryID}) response = routing.GetTransactionFromRelay(httpReq, &request, relayAPI, *userID) assert.Equal(t, http.StatusOK, response.Code) - jsonResponse = response.JSON.(gomatrixserverlib.RespGetRelayTransaction) + jsonResponse = response.JSON.(fclient.RespGetRelayTransaction) assert.False(t, jsonResponse.EntriesQueued) assert.Equal(t, gomatrixserverlib.Transaction{}, jsonResponse.Transaction) @@ -189,28 +190,28 @@ func TestGetReturnsMultipleSavedTransactions(t *testing.T) { &db, nil, nil, nil, nil, false, "", true, ) - request := createQuery(*userID, gomatrixserverlib.RelayEntry{}) + request := createQuery(*userID, fclient.RelayEntry{}) response := routing.GetTransactionFromRelay(httpReq, &request, relayAPI, *userID) assert.Equal(t, http.StatusOK, response.Code) - jsonResponse := response.JSON.(gomatrixserverlib.RespGetRelayTransaction) + jsonResponse := response.JSON.(fclient.RespGetRelayTransaction) assert.True(t, jsonResponse.EntriesQueued) assert.Equal(t, transaction, jsonResponse.Transaction) - request = createQuery(*userID, gomatrixserverlib.RelayEntry{EntryID: jsonResponse.EntryID}) + request = createQuery(*userID, fclient.RelayEntry{EntryID: jsonResponse.EntryID}) response = routing.GetTransactionFromRelay(httpReq, &request, relayAPI, *userID) assert.Equal(t, http.StatusOK, response.Code) - jsonResponse = response.JSON.(gomatrixserverlib.RespGetRelayTransaction) + jsonResponse = response.JSON.(fclient.RespGetRelayTransaction) assert.True(t, jsonResponse.EntriesQueued) assert.Equal(t, transaction2, jsonResponse.Transaction) // And once more to clear the queue - request = createQuery(*userID, gomatrixserverlib.RelayEntry{EntryID: jsonResponse.EntryID}) + request = createQuery(*userID, fclient.RelayEntry{EntryID: jsonResponse.EntryID}) response = routing.GetTransactionFromRelay(httpReq, &request, relayAPI, *userID) assert.Equal(t, http.StatusOK, response.Code) - jsonResponse = response.JSON.(gomatrixserverlib.RespGetRelayTransaction) + jsonResponse = response.JSON.(fclient.RespGetRelayTransaction) assert.False(t, jsonResponse.EntriesQueued) assert.Equal(t, gomatrixserverlib.Transaction{}, jsonResponse.Transaction) diff --git a/relayapi/routing/sendrelay.go b/relayapi/routing/sendrelay.go index ce744cb498..84c241032e 100644 --- a/relayapi/routing/sendrelay.go +++ b/relayapi/routing/sendrelay.go @@ -21,6 +21,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/relayapi/api" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/matrix-org/util" "github.com/sirupsen/logrus" ) @@ -36,7 +37,7 @@ func SendTransactionToRelay( ) util.JSONResponse { logrus.Infof("Processing send_relay for %s", userID.Raw()) - var txnEvents gomatrixserverlib.RelayEvents + var txnEvents fclient.RelayEvents if err := json.Unmarshal(fedReq.Content(), &txnEvents); err != nil { logrus.Info("The request body could not be decoded into valid JSON." + err.Error()) return util.JSONResponse{ diff --git a/roomserver/api/wrapper.go b/roomserver/api/wrapper.go index f220560edf..5f74c7854a 100644 --- a/roomserver/api/wrapper.go +++ b/roomserver/api/wrapper.go @@ -18,6 +18,7 @@ import ( "context" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/matrix-org/util" "github.com/sirupsen/logrus" ) @@ -49,7 +50,7 @@ func SendEvents( func SendEventWithState( ctx context.Context, rsAPI InputRoomEventsAPI, virtualHost gomatrixserverlib.ServerName, kind Kind, - state *gomatrixserverlib.RespState, event *gomatrixserverlib.HeaderedEvent, + state *fclient.RespState, event *gomatrixserverlib.HeaderedEvent, origin gomatrixserverlib.ServerName, haveEventIDs map[string]bool, async bool, ) error { outliers := state.Events(event.RoomVersion) @@ -159,7 +160,7 @@ func IsServerBannedFromRoom(ctx context.Context, rsAPI FederationRoomserverAPI, // PopulatePublicRooms extracts PublicRoom information for all the provided room IDs. The IDs are not checked to see if they are visible in the // published room directory. // due to lots of switches -func PopulatePublicRooms(ctx context.Context, roomIDs []string, rsAPI QueryBulkStateContentAPI) ([]gomatrixserverlib.PublicRoom, error) { +func PopulatePublicRooms(ctx context.Context, roomIDs []string, rsAPI QueryBulkStateContentAPI) ([]fclient.PublicRoom, error) { avatarTuple := gomatrixserverlib.StateKeyTuple{EventType: "m.room.avatar", StateKey: ""} nameTuple := gomatrixserverlib.StateKeyTuple{EventType: "m.room.name", StateKey: ""} canonicalTuple := gomatrixserverlib.StateKeyTuple{EventType: gomatrixserverlib.MRoomCanonicalAlias, StateKey: ""} @@ -181,10 +182,10 @@ func PopulatePublicRooms(ctx context.Context, roomIDs []string, rsAPI QueryBulkS util.GetLogger(ctx).WithError(err).Error("QueryBulkStateContent failed") return nil, err } - chunk := make([]gomatrixserverlib.PublicRoom, len(roomIDs)) + chunk := make([]fclient.PublicRoom, len(roomIDs)) i := 0 for roomID, data := range stateRes.Rooms { - pub := gomatrixserverlib.PublicRoom{ + pub := fclient.PublicRoom{ RoomID: roomID, } joinCount := 0 diff --git a/roomserver/internal/input/input.go b/roomserver/internal/input/input.go index cc0c673d02..83aa9e90fe 100644 --- a/roomserver/internal/input/input.go +++ b/roomserver/internal/input/input.go @@ -24,6 +24,7 @@ import ( "time" userapi "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/Arceliar/phony" "github.com/getsentry/sentry-go" @@ -79,7 +80,7 @@ type Inputer struct { JetStream nats.JetStreamContext Durable nats.SubOpt ServerName gomatrixserverlib.ServerName - SigningIdentity *gomatrixserverlib.SigningIdentity + SigningIdentity *fclient.SigningIdentity FSAPI fedapi.RoomserverFederationAPI KeyRing gomatrixserverlib.JSONVerifier ACLs *acls.ServerACLs diff --git a/roomserver/internal/input/input_events.go b/roomserver/internal/input/input_events.go index 7c7a902f56..971befa076 100644 --- a/roomserver/internal/input/input_events.go +++ b/roomserver/internal/input/input_events.go @@ -27,6 +27,7 @@ import ( "github.com/tidwall/gjson" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/matrix-org/util" "github.com/prometheus/client_golang/prometheus" "github.com/sirupsen/logrus" @@ -646,7 +647,7 @@ func (r *Inputer) fetchAuthEvents( } var err error - var res gomatrixserverlib.RespEventAuth + var res fclient.RespEventAuth var found bool for _, serverName := range servers { // Request the entire auth chain for the event in question. This should diff --git a/roomserver/internal/input/input_missing.go b/roomserver/internal/input/input_missing.go index daef957f16..74b138741f 100644 --- a/roomserver/internal/input/input_missing.go +++ b/roomserver/internal/input/input_missing.go @@ -8,6 +8,7 @@ import ( "time" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/matrix-org/util" "github.com/sirupsen/logrus" @@ -517,10 +518,10 @@ func (t *missingStateReq) getMissingEvents(ctx context.Context, e *gomatrixserve t.hadEvent(ev.EventID) } - var missingResp *gomatrixserverlib.RespMissingEvents + var missingResp *fclient.RespMissingEvents for _, server := range t.servers { - var m gomatrixserverlib.RespMissingEvents - if m, err = t.federation.LookupMissingEvents(ctx, t.virtualHost, server, e.RoomID(), gomatrixserverlib.MissingEvents{ + var m fclient.RespMissingEvents + if m, err = t.federation.LookupMissingEvents(ctx, t.virtualHost, server, e.RoomID(), fclient.MissingEvents{ Limit: 20, // The latest event IDs that the sender already has. These are skipped when retrieving the previous events of latest_events. EarliestEvents: latestEvents, @@ -640,8 +641,12 @@ func (t *missingStateReq) lookupMissingStateViaState( if err != nil { return nil, err } + s := fclient.RespState{ + StateEvents: state.GetStateEvents(), + AuthEvents: state.GetAuthEvents(), + } // Check that the returned state is valid. - authEvents, stateEvents, err := state.Check(ctx, roomVersion, t.keys, nil) + authEvents, stateEvents, err := s.Check(ctx, roomVersion, t.keys, nil) if err != nil { return nil, err } @@ -652,11 +657,11 @@ func (t *missingStateReq) lookupMissingStateViaState( // Cache the results of this state lookup and deduplicate anything we already // have in the cache, freeing up memory. // We load these as trusted as we called state.Check before which loaded them as untrusted. - for i, evJSON := range state.AuthEvents { + for i, evJSON := range s.AuthEvents { ev, _ := gomatrixserverlib.NewEventFromTrustedJSON(evJSON, false, roomVersion) parsedState.AuthEvents[i] = t.cacheAndReturn(ev) } - for i, evJSON := range state.StateEvents { + for i, evJSON := range s.StateEvents { ev, _ := gomatrixserverlib.NewEventFromTrustedJSON(evJSON, false, roomVersion) parsedState.StateEvents[i] = t.cacheAndReturn(ev) } @@ -670,7 +675,7 @@ func (t *missingStateReq) lookupMissingStateViaStateIDs(ctx context.Context, roo t.log.Infof("lookupMissingStateViaStateIDs %s", eventID) // fetch the state event IDs at the time of the event - var stateIDs gomatrixserverlib.RespStateIDs + var stateIDs gomatrixserverlib.StateIDResponse var err error count := 0 totalctx, totalcancel := context.WithTimeout(ctx, time.Minute*5) @@ -688,7 +693,7 @@ func (t *missingStateReq) lookupMissingStateViaStateIDs(ctx context.Context, roo return nil, fmt.Errorf("t.federation.LookupStateIDs tried %d server(s), last error: %w", count, err) } // work out which auth/state IDs are missing - wantIDs := append(stateIDs.StateEventIDs, stateIDs.AuthEventIDs...) + wantIDs := append(stateIDs.GetStateEventIDs(), stateIDs.GetAuthEventIDs()...) missing := make(map[string]bool) var missingEventList []string t.haveEventsMutex.Lock() @@ -730,8 +735,8 @@ func (t *missingStateReq) lookupMissingStateViaStateIDs(ctx context.Context, roo t.log.WithFields(logrus.Fields{ "missing": missingCount, "event_id": eventID, - "total_state": len(stateIDs.StateEventIDs), - "total_auth_events": len(stateIDs.AuthEventIDs), + "total_state": len(stateIDs.GetStateEventIDs()), + "total_auth_events": len(stateIDs.GetAuthEventIDs()), }).Debug("Fetching all state at event") return t.lookupMissingStateViaState(ctx, roomID, eventID, roomVersion) } @@ -740,8 +745,8 @@ func (t *missingStateReq) lookupMissingStateViaStateIDs(ctx context.Context, roo t.log.WithFields(logrus.Fields{ "missing": missingCount, "event_id": eventID, - "total_state": len(stateIDs.StateEventIDs), - "total_auth_events": len(stateIDs.AuthEventIDs), + "total_state": len(stateIDs.GetStateEventIDs()), + "total_auth_events": len(stateIDs.GetAuthEventIDs()), "concurrent_requests": concurrentRequests, }).Debug("Fetching missing state at event") @@ -808,7 +813,7 @@ func (t *missingStateReq) lookupMissingStateViaStateIDs(ctx context.Context, roo } func (t *missingStateReq) createRespStateFromStateIDs( - stateIDs gomatrixserverlib.RespStateIDs, + stateIDs gomatrixserverlib.StateIDResponse, ) (*parsedRespState, error) { // nolint:unparam t.haveEventsMutex.Lock() defer t.haveEventsMutex.Unlock() @@ -816,18 +821,20 @@ func (t *missingStateReq) createRespStateFromStateIDs( // create a RespState response using the response to /state_ids as a guide respState := parsedRespState{} - for i := range stateIDs.StateEventIDs { - ev, ok := t.haveEvents[stateIDs.StateEventIDs[i]] + stateEventIDs := stateIDs.GetStateEventIDs() + authEventIDs := stateIDs.GetAuthEventIDs() + for i := range stateEventIDs { + ev, ok := t.haveEvents[stateEventIDs[i]] if !ok { - logrus.Tracef("Missing state event in createRespStateFromStateIDs: %s", stateIDs.StateEventIDs[i]) + logrus.Tracef("Missing state event in createRespStateFromStateIDs: %s", stateEventIDs[i]) continue } respState.StateEvents = append(respState.StateEvents, ev) } - for i := range stateIDs.AuthEventIDs { - ev, ok := t.haveEvents[stateIDs.AuthEventIDs[i]] + for i := range authEventIDs { + ev, ok := t.haveEvents[authEventIDs[i]] if !ok { - logrus.Tracef("Missing auth event in createRespStateFromStateIDs: %s", stateIDs.AuthEventIDs[i]) + logrus.Tracef("Missing auth event in createRespStateFromStateIDs: %s", authEventIDs[i]) continue } respState.AuthEvents = append(respState.AuthEvents, ev) diff --git a/roomserver/internal/perform/perform_admin.go b/roomserver/internal/perform/perform_admin.go index 0f1249114a..f35e40bc91 100644 --- a/roomserver/internal/perform/perform_admin.go +++ b/roomserver/internal/perform/perform_admin.go @@ -322,7 +322,7 @@ func (r *Admin) PerformAdminDownloadState( stateEventMap := map[string]*gomatrixserverlib.Event{} for _, fwdExtremity := range fwdExtremities { - var state gomatrixserverlib.RespState + var state gomatrixserverlib.StateResponse state, err = r.Inputer.FSAPI.LookupState(ctx, r.Inputer.ServerName, req.ServerName, req.RoomID, fwdExtremity.EventID, roomInfo.RoomVersion) if err != nil { res.Error = &api.PerformError{ @@ -331,13 +331,13 @@ func (r *Admin) PerformAdminDownloadState( } return nil } - for _, authEvent := range state.AuthEvents.UntrustedEvents(roomInfo.RoomVersion) { + for _, authEvent := range state.GetAuthEvents().UntrustedEvents(roomInfo.RoomVersion) { if err = authEvent.VerifyEventSignatures(ctx, r.Inputer.KeyRing); err != nil { continue } authEventMap[authEvent.EventID()] = authEvent } - for _, stateEvent := range state.StateEvents.UntrustedEvents(roomInfo.RoomVersion) { + for _, stateEvent := range state.GetStateEvents().UntrustedEvents(roomInfo.RoomVersion) { if err = stateEvent.VerifyEventSignatures(ctx, r.Inputer.KeyRing); err != nil { continue } diff --git a/setup/base/base.go b/setup/base/base.go index e5fd6fed11..d6c3501098 100644 --- a/setup/base/base.go +++ b/setup/base/base.go @@ -32,7 +32,7 @@ import ( "time" sentryhttp "github.com/getsentry/sentry-go/http" - "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/prometheus/client_golang/prometheus/promhttp" "go.uber.org/atomic" @@ -54,42 +54,42 @@ const HTTPServerTimeout = time.Minute * 5 // CreateClient creates a new client (normally used for media fetch requests). // Should only be called once per component. -func CreateClient(cfg *config.Dendrite, dnsCache *gomatrixserverlib.DNSCache) *gomatrixserverlib.Client { +func CreateClient(cfg *config.Dendrite, dnsCache *fclient.DNSCache) *fclient.Client { if cfg.Global.DisableFederation { - return gomatrixserverlib.NewClient( - gomatrixserverlib.WithTransport(noOpHTTPTransport), + return fclient.NewClient( + fclient.WithTransport(noOpHTTPTransport), ) } - opts := []gomatrixserverlib.ClientOption{ - gomatrixserverlib.WithSkipVerify(cfg.FederationAPI.DisableTLSValidation), - gomatrixserverlib.WithWellKnownSRVLookups(true), + opts := []fclient.ClientOption{ + fclient.WithSkipVerify(cfg.FederationAPI.DisableTLSValidation), + fclient.WithWellKnownSRVLookups(true), } if cfg.Global.DNSCache.Enabled && dnsCache != nil { - opts = append(opts, gomatrixserverlib.WithDNSCache(dnsCache)) + opts = append(opts, fclient.WithDNSCache(dnsCache)) } - client := gomatrixserverlib.NewClient(opts...) + client := fclient.NewClient(opts...) client.SetUserAgent(fmt.Sprintf("Dendrite/%s", internal.VersionString())) return client } // CreateFederationClient creates a new federation client. Should only be called // once per component. -func CreateFederationClient(cfg *config.Dendrite, dnsCache *gomatrixserverlib.DNSCache) *gomatrixserverlib.FederationClient { +func CreateFederationClient(cfg *config.Dendrite, dnsCache *fclient.DNSCache) *fclient.FederationClient { identities := cfg.Global.SigningIdentities() if cfg.Global.DisableFederation { - return gomatrixserverlib.NewFederationClient( - identities, gomatrixserverlib.WithTransport(noOpHTTPTransport), + return fclient.NewFederationClient( + identities, fclient.WithTransport(noOpHTTPTransport), ) } - opts := []gomatrixserverlib.ClientOption{ - gomatrixserverlib.WithTimeout(time.Minute * 5), - gomatrixserverlib.WithSkipVerify(cfg.FederationAPI.DisableTLSValidation), - gomatrixserverlib.WithKeepAlives(!cfg.FederationAPI.DisableHTTPKeepalives), + opts := []fclient.ClientOption{ + fclient.WithTimeout(time.Minute * 5), + fclient.WithSkipVerify(cfg.FederationAPI.DisableTLSValidation), + fclient.WithKeepAlives(!cfg.FederationAPI.DisableHTTPKeepalives), } if cfg.Global.DNSCache.Enabled { - opts = append(opts, gomatrixserverlib.WithDNSCache(dnsCache)) + opts = append(opts, fclient.WithDNSCache(dnsCache)) } - client := gomatrixserverlib.NewFederationClient( + client := fclient.NewFederationClient( identities, opts..., ) client.SetUserAgent(fmt.Sprintf("Dendrite/%s", internal.VersionString())) diff --git a/setup/config/config_global.go b/setup/config/config_global.go index 7d3ab6a400..0687e9d351 100644 --- a/setup/config/config_global.go +++ b/setup/config/config_global.go @@ -8,13 +8,14 @@ import ( "time" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/fclient" "golang.org/x/crypto/ed25519" ) type Global struct { // Signing identity contains the server name, private key and key ID of // the deployment. - gomatrixserverlib.SigningIdentity `yaml:",inline"` + fclient.SigningIdentity `yaml:",inline"` // The secondary server names, used for virtual hosting. VirtualHosts []*VirtualHost `yaml:"-"` @@ -167,7 +168,7 @@ func (c *Global) VirtualHostForHTTPHost(serverName gomatrixserverlib.ServerName) return nil } -func (c *Global) SigningIdentityFor(serverName gomatrixserverlib.ServerName) (*gomatrixserverlib.SigningIdentity, error) { +func (c *Global) SigningIdentityFor(serverName gomatrixserverlib.ServerName) (*fclient.SigningIdentity, error) { for _, id := range c.SigningIdentities() { if id.ServerName == serverName { return id, nil @@ -176,8 +177,8 @@ func (c *Global) SigningIdentityFor(serverName gomatrixserverlib.ServerName) (*g return nil, fmt.Errorf("no signing identity for %q", serverName) } -func (c *Global) SigningIdentities() []*gomatrixserverlib.SigningIdentity { - identities := make([]*gomatrixserverlib.SigningIdentity, 0, len(c.VirtualHosts)+1) +func (c *Global) SigningIdentities() []*fclient.SigningIdentity { + identities := make([]*fclient.SigningIdentity, 0, len(c.VirtualHosts)+1) identities = append(identities, &c.SigningIdentity) for _, v := range c.VirtualHosts { identities = append(identities, &v.SigningIdentity) @@ -188,7 +189,7 @@ func (c *Global) SigningIdentities() []*gomatrixserverlib.SigningIdentity { type VirtualHost struct { // Signing identity contains the server name, private key and key ID of // the virtual host. - gomatrixserverlib.SigningIdentity `yaml:",inline"` + fclient.SigningIdentity `yaml:",inline"` // Path to the private key. If not specified, the default global private key // will be used instead. diff --git a/setup/config/config_test.go b/setup/config/config_test.go index 79407f30d5..a0509aafbf 100644 --- a/setup/config/config_test.go +++ b/setup/config/config_test.go @@ -20,6 +20,7 @@ import ( "testing" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/sirupsen/logrus" "gopkg.in/yaml.v2" ) @@ -275,7 +276,7 @@ func Test_SigningIdentityFor(t *testing.T) { name string virtualHosts []*VirtualHost serverName gomatrixserverlib.ServerName - want *gomatrixserverlib.SigningIdentity + want *fclient.SigningIdentity wantErr bool }{ { @@ -290,23 +291,23 @@ func Test_SigningIdentityFor(t *testing.T) { { name: "found identity", serverName: gomatrixserverlib.ServerName("main"), - want: &gomatrixserverlib.SigningIdentity{ServerName: "main"}, + want: &fclient.SigningIdentity{ServerName: "main"}, }, { name: "identity found on virtual hosts", serverName: gomatrixserverlib.ServerName("vh2"), virtualHosts: []*VirtualHost{ - {SigningIdentity: gomatrixserverlib.SigningIdentity{ServerName: "vh1"}}, - {SigningIdentity: gomatrixserverlib.SigningIdentity{ServerName: "vh2"}}, + {SigningIdentity: fclient.SigningIdentity{ServerName: "vh1"}}, + {SigningIdentity: fclient.SigningIdentity{ServerName: "vh2"}}, }, - want: &gomatrixserverlib.SigningIdentity{ServerName: "vh2"}, + want: &fclient.SigningIdentity{ServerName: "vh2"}, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { c := &Global{ VirtualHosts: tt.virtualHosts, - SigningIdentity: gomatrixserverlib.SigningIdentity{ + SigningIdentity: fclient.SigningIdentity{ ServerName: "main", }, } diff --git a/setup/monolith.go b/setup/monolith.go index 51e40070b6..e5af698538 100644 --- a/setup/monolith.go +++ b/setup/monolith.go @@ -34,6 +34,7 @@ import ( "github.com/matrix-org/dendrite/syncapi" userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/fclient" ) // Monolith represents an instantiation of all dependencies required to build @@ -41,8 +42,8 @@ import ( type Monolith struct { Config *config.Dendrite KeyRing *gomatrixserverlib.KeyRing - Client *gomatrixserverlib.Client - FedClient *gomatrixserverlib.FederationClient + Client *fclient.Client + FedClient *fclient.FederationClient AppserviceAPI appserviceAPI.AppServiceInternalAPI FederationAPI federationAPI.FederationInternalAPI diff --git a/setup/mscs/msc2836/msc2836.go b/setup/mscs/msc2836/msc2836.go index 2612f9117e..e1758920b3 100644 --- a/setup/mscs/msc2836/msc2836.go +++ b/setup/mscs/msc2836/msc2836.go @@ -37,6 +37,7 @@ import ( "github.com/matrix-org/dendrite/syncapi/synctypes" userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/matrix-org/util" ) @@ -85,7 +86,7 @@ type EventRelationshipResponse struct { } type MSC2836EventRelationshipsResponse struct { - gomatrixserverlib.MSC2836EventRelationshipsResponse + fclient.MSC2836EventRelationshipsResponse ParsedEvents []*gomatrixserverlib.Event ParsedAuthChain []*gomatrixserverlib.Event } @@ -399,7 +400,7 @@ func (rc *reqCtx) includeChildren(db Database, parentID string, limit int, recen serversToQuery := rc.getServersForEventID(parentID) var result *MSC2836EventRelationshipsResponse for _, srv := range serversToQuery { - res, err := rc.fsAPI.MSC2836EventRelationships(rc.ctx, rc.serverName, srv, gomatrixserverlib.MSC2836EventRelationshipsRequest{ + res, err := rc.fsAPI.MSC2836EventRelationships(rc.ctx, rc.serverName, srv, fclient.MSC2836EventRelationshipsRequest{ EventID: parentID, Direction: "down", Limit: 100, @@ -486,7 +487,7 @@ func walkThread( // MSC2836EventRelationships performs an /event_relationships request to a remote server func (rc *reqCtx) MSC2836EventRelationships(eventID string, srv gomatrixserverlib.ServerName, ver gomatrixserverlib.RoomVersion) (*MSC2836EventRelationshipsResponse, error) { - res, err := rc.fsAPI.MSC2836EventRelationships(rc.ctx, rc.serverName, srv, gomatrixserverlib.MSC2836EventRelationshipsRequest{ + res, err := rc.fsAPI.MSC2836EventRelationships(rc.ctx, rc.serverName, srv, fclient.MSC2836EventRelationshipsRequest{ EventID: eventID, DepthFirst: rc.req.DepthFirst, Direction: rc.req.Direction, @@ -653,7 +654,7 @@ func (rc *reqCtx) injectResponseToRoomserver(res *MSC2836EventRelationshipsRespo messageEvents = append(messageEvents, ev) } } - respState := gomatrixserverlib.RespState{ + respState := fclient.RespState{ AuthEvents: res.AuthChain, StateEvents: stateEvents, } diff --git a/setup/mscs/msc2946/msc2946.go b/setup/mscs/msc2946/msc2946.go index d33897b936..965af92071 100644 --- a/setup/mscs/msc2946/msc2946.go +++ b/setup/mscs/msc2946/msc2946.go @@ -36,6 +36,7 @@ import ( "github.com/matrix-org/dendrite/setup/config" userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/matrix-org/util" "github.com/tidwall/gjson" ) @@ -48,8 +49,8 @@ const ( ) type MSC2946ClientResponse struct { - Rooms []gomatrixserverlib.MSC2946Room `json:"rooms"` - NextBatch string `json:"next_batch,omitempty"` + Rooms []fclient.MSC2946Room `json:"rooms"` + NextBatch string `json:"next_batch,omitempty"` } // Enable this MSC @@ -222,7 +223,7 @@ func (w *walker) walk() util.JSONResponse { } } - var discoveredRooms []gomatrixserverlib.MSC2946Room + var discoveredRooms []fclient.MSC2946Room var cache *paginationInfo if w.paginationToken != "" { @@ -275,7 +276,7 @@ func (w *walker) walk() util.JSONResponse { } // Collect rooms/events to send back (either locally or fetched via federation) - var discoveredChildEvents []gomatrixserverlib.MSC2946StrippedEvent + var discoveredChildEvents []fclient.MSC2946StrippedEvent // If we know about this room and the caller is authorised (joined/world_readable) then pull // events locally @@ -306,7 +307,7 @@ func (w *walker) walk() util.JSONResponse { pubRoom := w.publicRoomsChunk(rv.roomID) - discoveredRooms = append(discoveredRooms, gomatrixserverlib.MSC2946Room{ + discoveredRooms = append(discoveredRooms, fclient.MSC2946Room{ PublicRoom: *pubRoom, RoomType: roomType, ChildrenState: events, @@ -379,7 +380,7 @@ func (w *walker) walk() util.JSONResponse { } return util.JSONResponse{ Code: 200, - JSON: gomatrixserverlib.MSC2946SpacesResponse{ + JSON: fclient.MSC2946SpacesResponse{ Room: discoveredRooms[0], Children: discoveredRooms[1:], }, @@ -402,7 +403,7 @@ func (w *walker) stateEvent(roomID, evType, stateKey string) *gomatrixserverlib. return queryRes.StateEvents[tuple] } -func (w *walker) publicRoomsChunk(roomID string) *gomatrixserverlib.PublicRoom { +func (w *walker) publicRoomsChunk(roomID string) *fclient.PublicRoom { pubRooms, err := roomserver.PopulatePublicRooms(w.ctx, []string{roomID}, w.rsAPI) if err != nil { util.GetLogger(w.ctx).WithError(err).Error("failed to PopulatePublicRooms") @@ -416,7 +417,7 @@ func (w *walker) publicRoomsChunk(roomID string) *gomatrixserverlib.PublicRoom { // federatedRoomInfo returns more of the spaces graph from another server. Returns nil if this was // unsuccessful. -func (w *walker) federatedRoomInfo(roomID string, vias []string) *gomatrixserverlib.MSC2946SpacesResponse { +func (w *walker) federatedRoomInfo(roomID string, vias []string) *fclient.MSC2946SpacesResponse { // only do federated requests for client requests if w.caller == nil { return nil @@ -440,12 +441,12 @@ func (w *walker) federatedRoomInfo(roomID string, vias []string) *gomatrixserver } // ensure nil slices are empty as we send this to the client sometimes if res.Room.ChildrenState == nil { - res.Room.ChildrenState = []gomatrixserverlib.MSC2946StrippedEvent{} + res.Room.ChildrenState = []fclient.MSC2946StrippedEvent{} } for i := 0; i < len(res.Children); i++ { child := res.Children[i] if child.ChildrenState == nil { - child.ChildrenState = []gomatrixserverlib.MSC2946StrippedEvent{} + child.ChildrenState = []fclient.MSC2946StrippedEvent{} } res.Children[i] = child } @@ -653,7 +654,7 @@ func (w *walker) restrictedJoinRuleAllowedRooms(joinRuleEv *gomatrixserverlib.He } // references returns all child references pointing to or from this room. -func (w *walker) childReferences(roomID string) ([]gomatrixserverlib.MSC2946StrippedEvent, error) { +func (w *walker) childReferences(roomID string) ([]fclient.MSC2946StrippedEvent, error) { createTuple := gomatrixserverlib.StateKeyTuple{ EventType: gomatrixserverlib.MRoomCreate, StateKey: "", @@ -678,12 +679,12 @@ func (w *walker) childReferences(roomID string) ([]gomatrixserverlib.MSC2946Stri // escape the `.`s so gjson doesn't think it's nested roomType := gjson.GetBytes(res.StateEvents[createTuple].Content(), strings.ReplaceAll(ConstCreateEventContentKey, ".", `\.`)).Str if roomType != ConstCreateEventContentValueSpace { - return []gomatrixserverlib.MSC2946StrippedEvent{}, nil + return []fclient.MSC2946StrippedEvent{}, nil } } delete(res.StateEvents, createTuple) - el := make([]gomatrixserverlib.MSC2946StrippedEvent, 0, len(res.StateEvents)) + el := make([]fclient.MSC2946StrippedEvent, 0, len(res.StateEvents)) for _, ev := range res.StateEvents { content := gjson.ParseBytes(ev.Content()) // only return events that have a `via` key as per MSC1772 @@ -720,11 +721,11 @@ func (s set) isSet(val string) bool { return ok } -func stripped(ev *gomatrixserverlib.Event) *gomatrixserverlib.MSC2946StrippedEvent { +func stripped(ev *gomatrixserverlib.Event) *fclient.MSC2946StrippedEvent { if ev.StateKey() == nil { return nil } - return &gomatrixserverlib.MSC2946StrippedEvent{ + return &fclient.MSC2946StrippedEvent{ Type: ev.Type(), StateKey: *ev.StateKey(), Content: ev.Content(), diff --git a/userapi/api/api.go b/userapi/api/api.go index f65f81ad3b..ba1c374f1b 100644 --- a/userapi/api/api.go +++ b/userapi/api/api.go @@ -24,6 +24,7 @@ import ( "github.com/matrix-org/dendrite/syncapi/synctypes" "github.com/matrix-org/dendrite/userapi/types" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/internal/pushrules" @@ -719,9 +720,9 @@ type OutputCrossSigningKeyUpdate struct { } type CrossSigningKeyUpdate struct { - MasterKey *gomatrixserverlib.CrossSigningKey `json:"master_key,omitempty"` - SelfSigningKey *gomatrixserverlib.CrossSigningKey `json:"self_signing_key,omitempty"` - UserID string `json:"user_id"` + MasterKey *fclient.CrossSigningKey `json:"master_key,omitempty"` + SelfSigningKey *fclient.CrossSigningKey `json:"self_signing_key,omitempty"` + UserID string `json:"user_id"` } // DeviceKeysEqual returns true if the device keys updates contain the @@ -854,7 +855,7 @@ type PerformClaimKeysResponse struct { } type PerformUploadDeviceKeysRequest struct { - gomatrixserverlib.CrossSigningKeys + fclient.CrossSigningKeys // The user that uploaded the key, should be populated by the clientapi. UserID string } @@ -864,7 +865,7 @@ type PerformUploadDeviceKeysResponse struct { } type PerformUploadDeviceSignaturesRequest struct { - Signatures map[string]map[gomatrixserverlib.KeyID]gomatrixserverlib.CrossSigningForKeyOrDevice + Signatures map[string]map[gomatrixserverlib.KeyID]fclient.CrossSigningForKeyOrDevice // The user that uploaded the sig, should be populated by the clientapi. UserID string } @@ -888,9 +889,9 @@ type QueryKeysResponse struct { // Map of user_id to device_id to device_key DeviceKeys map[string]map[string]json.RawMessage // Maps of user_id to cross signing key - MasterKeys map[string]gomatrixserverlib.CrossSigningKey - SelfSigningKeys map[string]gomatrixserverlib.CrossSigningKey - UserSigningKeys map[string]gomatrixserverlib.CrossSigningKey + MasterKeys map[string]fclient.CrossSigningKey + SelfSigningKeys map[string]fclient.CrossSigningKey + UserSigningKeys map[string]fclient.CrossSigningKey // Set if there was a fatal error processing this query Error *KeyError } @@ -945,11 +946,11 @@ type QuerySignaturesResponse struct { // A map of target user ID -> target key/device ID -> origin user ID -> origin key/device ID -> signatures Signatures map[string]map[gomatrixserverlib.KeyID]types.CrossSigningSigMap // A map of target user ID -> cross-signing master key - MasterKeys map[string]gomatrixserverlib.CrossSigningKey + MasterKeys map[string]fclient.CrossSigningKey // A map of target user ID -> cross-signing self-signing key - SelfSigningKeys map[string]gomatrixserverlib.CrossSigningKey + SelfSigningKeys map[string]fclient.CrossSigningKey // A map of target user ID -> cross-signing user-signing key - UserSigningKeys map[string]gomatrixserverlib.CrossSigningKey + UserSigningKeys map[string]fclient.CrossSigningKey // The request error, if any Error *KeyError } diff --git a/userapi/consumers/signingkeyupdate.go b/userapi/consumers/signingkeyupdate.go index f4ff017dbb..006ccb728f 100644 --- a/userapi/consumers/signingkeyupdate.go +++ b/userapi/consumers/signingkeyupdate.go @@ -19,6 +19,7 @@ import ( "encoding/json" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/nats-io/nats.go" "github.com/sirupsen/logrus" @@ -86,7 +87,7 @@ func (t *SigningKeyUpdateConsumer) onMessage(ctx context.Context, msgs []*nats.M return true } - keys := gomatrixserverlib.CrossSigningKeys{} + keys := fclient.CrossSigningKeys{} if updatePayload.MasterKey != nil { keys.MasterKey = *updatePayload.MasterKey } diff --git a/userapi/internal/cross_signing.go b/userapi/internal/cross_signing.go index 8b9704d1b0..23b6207e23 100644 --- a/userapi/internal/cross_signing.go +++ b/userapi/internal/cross_signing.go @@ -25,11 +25,12 @@ import ( "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/dendrite/userapi/types" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/sirupsen/logrus" "golang.org/x/crypto/curve25519" ) -func sanityCheckKey(key gomatrixserverlib.CrossSigningKey, userID string, purpose gomatrixserverlib.CrossSigningKeyPurpose) error { +func sanityCheckKey(key fclient.CrossSigningKey, userID string, purpose fclient.CrossSigningKeyPurpose) error { // Is there exactly one key? if len(key.Keys) != 1 { return fmt.Errorf("should contain exactly one key") @@ -105,12 +106,12 @@ func sanityCheckKey(key gomatrixserverlib.CrossSigningKey, userID string, purpos // nolint:gocyclo func (a *UserInternalAPI) PerformUploadDeviceKeys(ctx context.Context, req *api.PerformUploadDeviceKeysRequest, res *api.PerformUploadDeviceKeysResponse) error { // Find the keys to store. - byPurpose := map[gomatrixserverlib.CrossSigningKeyPurpose]gomatrixserverlib.CrossSigningKey{} + byPurpose := map[fclient.CrossSigningKeyPurpose]fclient.CrossSigningKey{} toStore := types.CrossSigningKeyMap{} hasMasterKey := false if len(req.MasterKey.Keys) > 0 { - if err := sanityCheckKey(req.MasterKey, req.UserID, gomatrixserverlib.CrossSigningKeyPurposeMaster); err != nil { + if err := sanityCheckKey(req.MasterKey, req.UserID, fclient.CrossSigningKeyPurposeMaster); err != nil { res.Error = &api.KeyError{ Err: "Master key sanity check failed: " + err.Error(), IsInvalidParam: true, @@ -118,15 +119,15 @@ func (a *UserInternalAPI) PerformUploadDeviceKeys(ctx context.Context, req *api. return nil } - byPurpose[gomatrixserverlib.CrossSigningKeyPurposeMaster] = req.MasterKey + byPurpose[fclient.CrossSigningKeyPurposeMaster] = req.MasterKey for _, key := range req.MasterKey.Keys { // iterates once, see sanityCheckKey - toStore[gomatrixserverlib.CrossSigningKeyPurposeMaster] = key + toStore[fclient.CrossSigningKeyPurposeMaster] = key } hasMasterKey = true } if len(req.SelfSigningKey.Keys) > 0 { - if err := sanityCheckKey(req.SelfSigningKey, req.UserID, gomatrixserverlib.CrossSigningKeyPurposeSelfSigning); err != nil { + if err := sanityCheckKey(req.SelfSigningKey, req.UserID, fclient.CrossSigningKeyPurposeSelfSigning); err != nil { res.Error = &api.KeyError{ Err: "Self-signing key sanity check failed: " + err.Error(), IsInvalidParam: true, @@ -134,14 +135,14 @@ func (a *UserInternalAPI) PerformUploadDeviceKeys(ctx context.Context, req *api. return nil } - byPurpose[gomatrixserverlib.CrossSigningKeyPurposeSelfSigning] = req.SelfSigningKey + byPurpose[fclient.CrossSigningKeyPurposeSelfSigning] = req.SelfSigningKey for _, key := range req.SelfSigningKey.Keys { // iterates once, see sanityCheckKey - toStore[gomatrixserverlib.CrossSigningKeyPurposeSelfSigning] = key + toStore[fclient.CrossSigningKeyPurposeSelfSigning] = key } } if len(req.UserSigningKey.Keys) > 0 { - if err := sanityCheckKey(req.UserSigningKey, req.UserID, gomatrixserverlib.CrossSigningKeyPurposeUserSigning); err != nil { + if err := sanityCheckKey(req.UserSigningKey, req.UserID, fclient.CrossSigningKeyPurposeUserSigning); err != nil { res.Error = &api.KeyError{ Err: "User-signing key sanity check failed: " + err.Error(), IsInvalidParam: true, @@ -149,9 +150,9 @@ func (a *UserInternalAPI) PerformUploadDeviceKeys(ctx context.Context, req *api. return nil } - byPurpose[gomatrixserverlib.CrossSigningKeyPurposeUserSigning] = req.UserSigningKey + byPurpose[fclient.CrossSigningKeyPurposeUserSigning] = req.UserSigningKey for _, key := range req.UserSigningKey.Keys { // iterates once, see sanityCheckKey - toStore[gomatrixserverlib.CrossSigningKeyPurposeUserSigning] = key + toStore[fclient.CrossSigningKeyPurposeUserSigning] = key } } @@ -180,7 +181,7 @@ func (a *UserInternalAPI) PerformUploadDeviceKeys(ctx context.Context, req *api. // If we still can't find a master key for the user then stop the upload. // This satisfies the "Fails to upload self-signing key without master key" test. if !hasMasterKey { - if _, hasMasterKey = existingKeys[gomatrixserverlib.CrossSigningKeyPurposeMaster]; !hasMasterKey { + if _, hasMasterKey = existingKeys[fclient.CrossSigningKeyPurposeMaster]; !hasMasterKey { res.Error = &api.KeyError{ Err: "No master key was found", IsMissingParam: true, @@ -191,10 +192,10 @@ func (a *UserInternalAPI) PerformUploadDeviceKeys(ctx context.Context, req *api. // Check if anything actually changed compared to what we have in the database. changed := false - for _, purpose := range []gomatrixserverlib.CrossSigningKeyPurpose{ - gomatrixserverlib.CrossSigningKeyPurposeMaster, - gomatrixserverlib.CrossSigningKeyPurposeSelfSigning, - gomatrixserverlib.CrossSigningKeyPurposeUserSigning, + for _, purpose := range []fclient.CrossSigningKeyPurpose{ + fclient.CrossSigningKeyPurposeMaster, + fclient.CrossSigningKeyPurposeSelfSigning, + fclient.CrossSigningKeyPurposeUserSigning, } { old, gotOld := existingKeys[purpose] new, gotNew := toStore[purpose] @@ -248,10 +249,10 @@ func (a *UserInternalAPI) PerformUploadDeviceKeys(ctx context.Context, req *api. update := api.CrossSigningKeyUpdate{ UserID: req.UserID, } - if mk, ok := byPurpose[gomatrixserverlib.CrossSigningKeyPurposeMaster]; ok { + if mk, ok := byPurpose[fclient.CrossSigningKeyPurposeMaster]; ok { update.MasterKey = &mk } - if ssk, ok := byPurpose[gomatrixserverlib.CrossSigningKeyPurposeSelfSigning]; ok { + if ssk, ok := byPurpose[fclient.CrossSigningKeyPurposeSelfSigning]; ok { update.SelfSigningKey = &ssk } if update.MasterKey == nil && update.SelfSigningKey == nil { @@ -279,36 +280,36 @@ func (a *UserInternalAPI) PerformUploadDeviceSignatures(ctx context.Context, req } _ = a.QueryKeys(ctx, queryReq, queryRes) - selfSignatures := map[string]map[gomatrixserverlib.KeyID]gomatrixserverlib.CrossSigningForKeyOrDevice{} - otherSignatures := map[string]map[gomatrixserverlib.KeyID]gomatrixserverlib.CrossSigningForKeyOrDevice{} + selfSignatures := map[string]map[gomatrixserverlib.KeyID]fclient.CrossSigningForKeyOrDevice{} + otherSignatures := map[string]map[gomatrixserverlib.KeyID]fclient.CrossSigningForKeyOrDevice{} // Sort signatures into two groups: one where people have signed their own // keys and one where people have signed someone elses for userID, forUserID := range req.Signatures { for keyID, keyOrDevice := range forUserID { switch key := keyOrDevice.CrossSigningBody.(type) { - case *gomatrixserverlib.CrossSigningKey: + case *fclient.CrossSigningKey: if key.UserID == req.UserID { if _, ok := selfSignatures[userID]; !ok { - selfSignatures[userID] = map[gomatrixserverlib.KeyID]gomatrixserverlib.CrossSigningForKeyOrDevice{} + selfSignatures[userID] = map[gomatrixserverlib.KeyID]fclient.CrossSigningForKeyOrDevice{} } selfSignatures[userID][keyID] = keyOrDevice } else { if _, ok := otherSignatures[userID]; !ok { - otherSignatures[userID] = map[gomatrixserverlib.KeyID]gomatrixserverlib.CrossSigningForKeyOrDevice{} + otherSignatures[userID] = map[gomatrixserverlib.KeyID]fclient.CrossSigningForKeyOrDevice{} } otherSignatures[userID][keyID] = keyOrDevice } - case *gomatrixserverlib.DeviceKeys: + case *fclient.DeviceKeys: if key.UserID == req.UserID { if _, ok := selfSignatures[userID]; !ok { - selfSignatures[userID] = map[gomatrixserverlib.KeyID]gomatrixserverlib.CrossSigningForKeyOrDevice{} + selfSignatures[userID] = map[gomatrixserverlib.KeyID]fclient.CrossSigningForKeyOrDevice{} } selfSignatures[userID][keyID] = keyOrDevice } else { if _, ok := otherSignatures[userID]; !ok { - otherSignatures[userID] = map[gomatrixserverlib.KeyID]gomatrixserverlib.CrossSigningForKeyOrDevice{} + otherSignatures[userID] = map[gomatrixserverlib.KeyID]fclient.CrossSigningForKeyOrDevice{} } otherSignatures[userID][keyID] = keyOrDevice } @@ -354,7 +355,7 @@ func (a *UserInternalAPI) PerformUploadDeviceSignatures(ctx context.Context, req func (a *UserInternalAPI) processSelfSignatures( ctx context.Context, - signatures map[string]map[gomatrixserverlib.KeyID]gomatrixserverlib.CrossSigningForKeyOrDevice, + signatures map[string]map[gomatrixserverlib.KeyID]fclient.CrossSigningForKeyOrDevice, ) error { // Here we will process: // * The user signing their own devices using their self-signing key @@ -363,7 +364,7 @@ func (a *UserInternalAPI) processSelfSignatures( for targetUserID, forTargetUserID := range signatures { for targetKeyID, signature := range forTargetUserID { switch sig := signature.CrossSigningBody.(type) { - case *gomatrixserverlib.CrossSigningKey: + case *fclient.CrossSigningKey: for keyID := range sig.Keys { split := strings.SplitN(string(keyID), ":", 2) if len(split) > 1 && gomatrixserverlib.KeyID(split[1]) == targetKeyID { @@ -381,7 +382,7 @@ func (a *UserInternalAPI) processSelfSignatures( } } - case *gomatrixserverlib.DeviceKeys: + case *fclient.DeviceKeys: for originUserID, forOriginUserID := range sig.Signatures { for originKeyID, originSig := range forOriginUserID { if err := a.KeyDatabase.StoreCrossSigningSigsForTarget( @@ -403,7 +404,7 @@ func (a *UserInternalAPI) processSelfSignatures( func (a *UserInternalAPI) processOtherSignatures( ctx context.Context, userID string, queryRes *api.QueryKeysResponse, - signatures map[string]map[gomatrixserverlib.KeyID]gomatrixserverlib.CrossSigningForKeyOrDevice, + signatures map[string]map[gomatrixserverlib.KeyID]fclient.CrossSigningForKeyOrDevice, ) error { // Here we will process: // * A user signing someone else's master keys using their user-signing keys @@ -411,7 +412,7 @@ func (a *UserInternalAPI) processOtherSignatures( for targetUserID, forTargetUserID := range signatures { for _, signature := range forTargetUserID { switch sig := signature.CrossSigningBody.(type) { - case *gomatrixserverlib.CrossSigningKey: + case *fclient.CrossSigningKey: // Find the local copy of the master key. We'll use this to be // sure that the supplied stanza matches the key that we think it // should be. @@ -509,13 +510,13 @@ func (a *UserInternalAPI) crossSigningKeysFromDatabase( } switch keyType { - case gomatrixserverlib.CrossSigningKeyPurposeMaster: + case fclient.CrossSigningKeyPurposeMaster: res.MasterKeys[targetUserID] = key - case gomatrixserverlib.CrossSigningKeyPurposeSelfSigning: + case fclient.CrossSigningKeyPurposeSelfSigning: res.SelfSigningKeys[targetUserID] = key - case gomatrixserverlib.CrossSigningKeyPurposeUserSigning: + case fclient.CrossSigningKeyPurposeUserSigning: res.UserSigningKeys[targetUserID] = key } } @@ -534,21 +535,21 @@ func (a *UserInternalAPI) QuerySignatures(ctx context.Context, req *api.QuerySig for targetPurpose, targetKey := range keyMap { switch targetPurpose { - case gomatrixserverlib.CrossSigningKeyPurposeMaster: + case fclient.CrossSigningKeyPurposeMaster: if res.MasterKeys == nil { - res.MasterKeys = map[string]gomatrixserverlib.CrossSigningKey{} + res.MasterKeys = map[string]fclient.CrossSigningKey{} } res.MasterKeys[targetUserID] = targetKey - case gomatrixserverlib.CrossSigningKeyPurposeSelfSigning: + case fclient.CrossSigningKeyPurposeSelfSigning: if res.SelfSigningKeys == nil { - res.SelfSigningKeys = map[string]gomatrixserverlib.CrossSigningKey{} + res.SelfSigningKeys = map[string]fclient.CrossSigningKey{} } res.SelfSigningKeys[targetUserID] = targetKey - case gomatrixserverlib.CrossSigningKeyPurposeUserSigning: + case fclient.CrossSigningKeyPurposeUserSigning: if res.UserSigningKeys == nil { - res.UserSigningKeys = map[string]gomatrixserverlib.CrossSigningKey{} + res.UserSigningKeys = map[string]fclient.CrossSigningKey{} } res.UserSigningKeys[targetUserID] = targetKey } diff --git a/userapi/internal/device_list_update.go b/userapi/internal/device_list_update.go index 3b4dcf98e2..a274e1ae3b 100644 --- a/userapi/internal/device_list_update.go +++ b/userapi/internal/device_list_update.go @@ -25,6 +25,7 @@ import ( "time" rsapi "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/matrix-org/gomatrix" "github.com/matrix-org/gomatrixserverlib" @@ -508,12 +509,12 @@ func (u *DeviceListUpdater) processServerUser(ctx context.Context, serverName go } uploadRes := &api.PerformUploadDeviceKeysResponse{} if res.MasterKey != nil { - if err = sanityCheckKey(*res.MasterKey, userID, gomatrixserverlib.CrossSigningKeyPurposeMaster); err == nil { + if err = sanityCheckKey(*res.MasterKey, userID, fclient.CrossSigningKeyPurposeMaster); err == nil { uploadReq.MasterKey = *res.MasterKey } } if res.SelfSigningKey != nil { - if err = sanityCheckKey(*res.SelfSigningKey, userID, gomatrixserverlib.CrossSigningKeyPurposeSelfSigning); err == nil { + if err = sanityCheckKey(*res.SelfSigningKey, userID, fclient.CrossSigningKeyPurposeSelfSigning); err == nil { uploadReq.SelfSigningKey = *res.SelfSigningKey } } @@ -527,7 +528,7 @@ func (u *DeviceListUpdater) processServerUser(ctx context.Context, serverName go return defaultWaitTime, nil } -func (u *DeviceListUpdater) updateDeviceList(res *gomatrixserverlib.RespUserDevices) error { +func (u *DeviceListUpdater) updateDeviceList(res *fclient.RespUserDevices) error { ctx := context.Background() // we've got the keys, don't time out when persisting them to the database. keys := make([]api.DeviceMessage, len(res.Devices)) existingKeys := make([]api.DeviceMessage, len(res.Devices)) diff --git a/userapi/internal/device_list_update_test.go b/userapi/internal/device_list_update_test.go index c0965a2c28..47b31c6850 100644 --- a/userapi/internal/device_list_update_test.go +++ b/userapi/internal/device_list_update_test.go @@ -29,6 +29,7 @@ import ( "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/fclient" roomserver "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/setup/config" @@ -135,10 +136,10 @@ func (t *roundTripper) RoundTrip(req *http.Request) (*http.Response, error) { return t.fn(req) } -func newFedClient(tripper func(*http.Request) (*http.Response, error)) *gomatrixserverlib.FederationClient { +func newFedClient(tripper func(*http.Request) (*http.Response, error)) *fclient.FederationClient { _, pkey, _ := ed25519.GenerateKey(nil) - fedClient := gomatrixserverlib.NewFederationClient( - []*gomatrixserverlib.SigningIdentity{ + fedClient := fclient.NewFederationClient( + []*fclient.SigningIdentity{ { ServerName: gomatrixserverlib.ServerName("example.test"), KeyID: gomatrixserverlib.KeyID("ed25519:test"), @@ -146,8 +147,8 @@ func newFedClient(tripper func(*http.Request) (*http.Response, error)) *gomatrix }, }, ) - fedClient.Client = *gomatrixserverlib.NewClient( - gomatrixserverlib.WithTransport(&roundTripper{tripper}), + fedClient.Client = *fclient.NewClient( + fclient.WithTransport(&roundTripper{tripper}), ) return fedClient } diff --git a/userapi/internal/key_api.go b/userapi/internal/key_api.go index be816fe5d6..043028725f 100644 --- a/userapi/internal/key_api.go +++ b/userapi/internal/key_api.go @@ -24,6 +24,7 @@ import ( "time" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/matrix-org/util" "github.com/sirupsen/logrus" "github.com/tidwall/gjson" @@ -229,9 +230,9 @@ func (a *UserInternalAPI) PerformMarkAsStaleIfNeeded(ctx context.Context, req *a func (a *UserInternalAPI) QueryKeys(ctx context.Context, req *api.QueryKeysRequest, res *api.QueryKeysResponse) error { var respMu sync.Mutex res.DeviceKeys = make(map[string]map[string]json.RawMessage) - res.MasterKeys = make(map[string]gomatrixserverlib.CrossSigningKey) - res.SelfSigningKeys = make(map[string]gomatrixserverlib.CrossSigningKey) - res.UserSigningKeys = make(map[string]gomatrixserverlib.CrossSigningKey) + res.MasterKeys = make(map[string]fclient.CrossSigningKey) + res.SelfSigningKeys = make(map[string]fclient.CrossSigningKey) + res.UserSigningKeys = make(map[string]fclient.CrossSigningKey) res.Failures = make(map[string]interface{}) // make a map from domain to device keys @@ -362,7 +363,7 @@ func (a *UserInternalAPI) QueryKeys(ctx context.Context, req *api.QueryKeysReque if len(sigMap) == 0 { continue } - var deviceKey gomatrixserverlib.DeviceKeys + var deviceKey fclient.DeviceKeys if err = json.Unmarshal(key, &deviceKey); err != nil { continue } @@ -415,7 +416,7 @@ func (a *UserInternalAPI) queryRemoteKeys( ctx context.Context, timeout time.Duration, res *api.QueryKeysResponse, domainToDeviceKeys map[string]map[string][]string, domainToCrossSigningKeys map[string]map[string]struct{}, ) { - resultCh := make(chan *gomatrixserverlib.RespQueryKeys, len(domainToDeviceKeys)) + resultCh := make(chan *fclient.RespQueryKeys, len(domainToDeviceKeys)) // allows us to wait until all federation servers have been poked var wg sync.WaitGroup // mutex for writing directly to res (e.g failures) @@ -450,7 +451,7 @@ func (a *UserInternalAPI) queryRemoteKeys( close(resultCh) }() - processResult := func(result *gomatrixserverlib.RespQueryKeys) { + processResult := func(result *fclient.RespQueryKeys) { respMu.Lock() defer respMu.Unlock() for userID, nest := range result.DeviceKeys { @@ -483,7 +484,7 @@ func (a *UserInternalAPI) queryRemoteKeys( func (a *UserInternalAPI) queryRemoteKeysOnServer( ctx context.Context, serverName string, devKeys map[string][]string, crossSigningKeys map[string]struct{}, - wg *sync.WaitGroup, respMu *sync.Mutex, timeout time.Duration, resultCh chan<- *gomatrixserverlib.RespQueryKeys, + wg *sync.WaitGroup, respMu *sync.Mutex, timeout time.Duration, resultCh chan<- *fclient.RespQueryKeys, res *api.QueryKeysResponse, ) { defer wg.Done() diff --git a/userapi/storage/interface.go b/userapi/storage/interface.go index 278378861e..4ffb126a7a 100644 --- a/userapi/storage/interface.go +++ b/userapi/storage/interface.go @@ -20,6 +20,7 @@ import ( "errors" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/internal/pushrules" @@ -203,7 +204,7 @@ type KeyDatabase interface { // MarkDeviceListStale sets the stale bit for this user to isStale. MarkDeviceListStale(ctx context.Context, userID string, isStale bool) error - CrossSigningKeysForUser(ctx context.Context, userID string) (map[gomatrixserverlib.CrossSigningKeyPurpose]gomatrixserverlib.CrossSigningKey, error) + CrossSigningKeysForUser(ctx context.Context, userID string) (map[fclient.CrossSigningKeyPurpose]fclient.CrossSigningKey, error) CrossSigningKeysDataForUser(ctx context.Context, userID string) (types.CrossSigningKeyMap, error) CrossSigningSigsForTarget(ctx context.Context, originUserID, targetUserID string, targetKeyID gomatrixserverlib.KeyID) (types.CrossSigningSigMap, error) diff --git a/userapi/storage/postgres/cross_signing_keys_table.go b/userapi/storage/postgres/cross_signing_keys_table.go index c0ecbd3039..b6fe6d7210 100644 --- a/userapi/storage/postgres/cross_signing_keys_table.go +++ b/userapi/storage/postgres/cross_signing_keys_table.go @@ -24,6 +24,7 @@ import ( "github.com/matrix-org/dendrite/userapi/storage/tables" "github.com/matrix-org/dendrite/userapi/types" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/fclient" ) var crossSigningKeysSchema = ` @@ -89,7 +90,7 @@ func (s *crossSigningKeysStatements) SelectCrossSigningKeysForUser( } func (s *crossSigningKeysStatements) UpsertCrossSigningKeysForUser( - ctx context.Context, txn *sql.Tx, userID string, keyType gomatrixserverlib.CrossSigningKeyPurpose, keyData gomatrixserverlib.Base64Bytes, + ctx context.Context, txn *sql.Tx, userID string, keyType fclient.CrossSigningKeyPurpose, keyData gomatrixserverlib.Base64Bytes, ) error { keyTypeInt, ok := types.KeyTypePurposeToInt[keyType] if !ok { diff --git a/userapi/storage/shared/storage.go b/userapi/storage/shared/storage.go index d3272a032d..a03d022ad4 100644 --- a/userapi/storage/shared/storage.go +++ b/userapi/storage/shared/storage.go @@ -27,6 +27,7 @@ import ( "time" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/fclient" "golang.org/x/crypto/bcrypt" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" @@ -1026,17 +1027,17 @@ func (d *KeyDatabase) DeleteDeviceKeys(ctx context.Context, userID string, devic } // CrossSigningKeysForUser returns the latest known cross-signing keys for a user, if any. -func (d *KeyDatabase) CrossSigningKeysForUser(ctx context.Context, userID string) (map[gomatrixserverlib.CrossSigningKeyPurpose]gomatrixserverlib.CrossSigningKey, error) { +func (d *KeyDatabase) CrossSigningKeysForUser(ctx context.Context, userID string) (map[fclient.CrossSigningKeyPurpose]fclient.CrossSigningKey, error) { keyMap, err := d.CrossSigningKeysTable.SelectCrossSigningKeysForUser(ctx, nil, userID) if err != nil { return nil, fmt.Errorf("d.CrossSigningKeysTable.SelectCrossSigningKeysForUser: %w", err) } - results := map[gomatrixserverlib.CrossSigningKeyPurpose]gomatrixserverlib.CrossSigningKey{} + results := map[fclient.CrossSigningKeyPurpose]fclient.CrossSigningKey{} for purpose, key := range keyMap { keyID := gomatrixserverlib.KeyID("ed25519:" + key.Encode()) - result := gomatrixserverlib.CrossSigningKey{ + result := fclient.CrossSigningKey{ UserID: userID, - Usage: []gomatrixserverlib.CrossSigningKeyPurpose{purpose}, + Usage: []fclient.CrossSigningKeyPurpose{purpose}, Keys: map[gomatrixserverlib.KeyID]gomatrixserverlib.Base64Bytes{ keyID: key, }, diff --git a/userapi/storage/sqlite3/cross_signing_keys_table.go b/userapi/storage/sqlite3/cross_signing_keys_table.go index 10721fcc86..e1c45c4116 100644 --- a/userapi/storage/sqlite3/cross_signing_keys_table.go +++ b/userapi/storage/sqlite3/cross_signing_keys_table.go @@ -24,6 +24,7 @@ import ( "github.com/matrix-org/dendrite/userapi/storage/tables" "github.com/matrix-org/dendrite/userapi/types" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/fclient" ) var crossSigningKeysSchema = ` @@ -88,7 +89,7 @@ func (s *crossSigningKeysStatements) SelectCrossSigningKeysForUser( } func (s *crossSigningKeysStatements) UpsertCrossSigningKeysForUser( - ctx context.Context, txn *sql.Tx, userID string, keyType gomatrixserverlib.CrossSigningKeyPurpose, keyData gomatrixserverlib.Base64Bytes, + ctx context.Context, txn *sql.Tx, userID string, keyType fclient.CrossSigningKeyPurpose, keyData gomatrixserverlib.Base64Bytes, ) error { keyTypeInt, ok := types.KeyTypePurposeToInt[keyType] if !ok { diff --git a/userapi/storage/tables/interface.go b/userapi/storage/tables/interface.go index 693e73038f..2d1339282c 100644 --- a/userapi/storage/tables/interface.go +++ b/userapi/storage/tables/interface.go @@ -22,6 +22,7 @@ import ( "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/userapi/types" @@ -181,7 +182,7 @@ type StaleDeviceLists interface { type CrossSigningKeys interface { SelectCrossSigningKeysForUser(ctx context.Context, txn *sql.Tx, userID string) (r types.CrossSigningKeyMap, err error) - UpsertCrossSigningKeysForUser(ctx context.Context, txn *sql.Tx, userID string, keyType gomatrixserverlib.CrossSigningKeyPurpose, keyData gomatrixserverlib.Base64Bytes) error + UpsertCrossSigningKeysForUser(ctx context.Context, txn *sql.Tx, userID string, keyType fclient.CrossSigningKeyPurpose, keyData gomatrixserverlib.Base64Bytes) error } type CrossSigningSigs interface { diff --git a/userapi/types/storage.go b/userapi/types/storage.go index 7fb90454e8..a910f7f101 100644 --- a/userapi/types/storage.go +++ b/userapi/types/storage.go @@ -18,6 +18,7 @@ import ( "math" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/fclient" ) const ( @@ -29,22 +30,22 @@ const ( // KeyTypePurposeToInt maps a purpose to an integer, which is used in the // database to reduce the amount of space taken up by this column. -var KeyTypePurposeToInt = map[gomatrixserverlib.CrossSigningKeyPurpose]int16{ - gomatrixserverlib.CrossSigningKeyPurposeMaster: 1, - gomatrixserverlib.CrossSigningKeyPurposeSelfSigning: 2, - gomatrixserverlib.CrossSigningKeyPurposeUserSigning: 3, +var KeyTypePurposeToInt = map[fclient.CrossSigningKeyPurpose]int16{ + fclient.CrossSigningKeyPurposeMaster: 1, + fclient.CrossSigningKeyPurposeSelfSigning: 2, + fclient.CrossSigningKeyPurposeUserSigning: 3, } // KeyTypeIntToPurpose maps an integer to a purpose, which is used in the // database to reduce the amount of space taken up by this column. -var KeyTypeIntToPurpose = map[int16]gomatrixserverlib.CrossSigningKeyPurpose{ - 1: gomatrixserverlib.CrossSigningKeyPurposeMaster, - 2: gomatrixserverlib.CrossSigningKeyPurposeSelfSigning, - 3: gomatrixserverlib.CrossSigningKeyPurposeUserSigning, +var KeyTypeIntToPurpose = map[int16]fclient.CrossSigningKeyPurpose{ + 1: fclient.CrossSigningKeyPurposeMaster, + 2: fclient.CrossSigningKeyPurposeSelfSigning, + 3: fclient.CrossSigningKeyPurposeUserSigning, } // Map of purpose -> public key -type CrossSigningKeyMap map[gomatrixserverlib.CrossSigningKeyPurpose]gomatrixserverlib.Base64Bytes +type CrossSigningKeyMap map[fclient.CrossSigningKeyPurpose]gomatrixserverlib.Base64Bytes // Map of user ID -> key ID -> signature type CrossSigningSigMap map[string]map[gomatrixserverlib.KeyID]gomatrixserverlib.Base64Bytes diff --git a/userapi/userapi_test.go b/userapi/userapi_test.go index e29246ec14..9d068ca3b1 100644 --- a/userapi/userapi_test.go +++ b/userapi/userapi_test.go @@ -27,6 +27,7 @@ import ( "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/userapi/producers" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/matrix-org/util" "github.com/nats-io/nats.go" "golang.org/x/crypto/bcrypt" @@ -87,7 +88,7 @@ func MustMakeInternalAPI(t *testing.T, opts apiTestOpts, dbType test.DBType, pub t.Fatalf("failed to create key DB: %s", err) } - cfg.Global.SigningIdentity = gomatrixserverlib.SigningIdentity{ + cfg.Global.SigningIdentity = fclient.SigningIdentity{ ServerName: sName, } From ee57400afdf3aacdd2af834f9dfaf90395523ae2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 6 Apr 2023 09:55:26 +0100 Subject: [PATCH 35/42] Bump github.com/docker/docker from 20.10.19+incompatible to 20.10.24+incompatible (#3047) Bumps [github.com/docker/docker](https://github.com/docker/docker) from 20.10.19+incompatible to 20.10.24+incompatible.
Release notes

Sourced from github.com/docker/docker's releases.

v20.10.24

20.10.24

Bug fixes and enhancements

  • Fixed a number of issues that can cause Swarm encrypted overlay networks to fail to uphold their guarantees, addressing CVE-2023-28841, CVE-2023-28840, and CVE-2023-28842.
    • A lack of kernel support for encrypted overlay networks now reports as an error.
    • Encrypted overlay networks are eagerly set up, rather than waiting for multiple nodes to attach.
    • Encrypted overlay networks are now usable on Red Hat Enterprise Linux 9 through the use of the xt_bpf kernel module.
    • Users of Swarm overlay networks should review GHSA-vwm3-crmr-xfxw to ensure that unintentional exposure has not occurred.
  • Upgrade github.com/containerd/fifo to v1.1.0 to fix a potential panic moby/moby#45216.
  • Fix missing Bash completion for installed cli-plugins docker/cli#4091.

Packaging Updates

v20.10.23

Bug fixes and enhancements

  • Fix an issue where docker build would fail when using --add-host=host.docker.internal:host-gateway with BuildKit enabled moby/moby#44650.

  • Revert seccomp: block socket calls to AF_VSOCK in default profile moby/moby#44712. This change, while favorable from a security standpoint, caused a change in behavior for some use-cases. As such, we are reverting it to ensure stability and compatibility for the affected users.

    However, users of AF_VSOCK in containers should recognize that this (special) address family is not currently namespaced in any version of the Linux kernel, and may result in unexpected behavior, like containers communicating directly with host hypervisors.

    Future releases, will filter AF_VSOCK. Users who need to allow containers to communicate over the unnamespaced AF_VSOCK will need to turn off seccomp confinement or set a custom seccomp profile.

Packaging Updates

  • Update Docker Compose to v2.15.1.

... (truncated)

Commits
  • 5d6db84 Merge pull request from GHSA-232p-vwff-86mp
  • d2bc43a Merge pull request #45242 from neersighted/go1.19.7/20.10
  • 9aa5d55 update to go1.19.7
  • 83679bb Merge pull request #45216 from corhere/backport-20.10/containerd-fifo_v1.1
  • b4f0442 Merge pull request #45219 from vvoland/test-windows-execstartfails-2010
  • ba043e8 Merge pull request #44990 from thaJeztah/20.10_backport_update_go1.19
  • b56fe59 integration-cli: Enable TestExecStartFails on Windows
  • d9433ee Merge pull request #45197 from vvoland/integration-restart-race-2010
  • a9c02c2 Upgrade containerd/fifo to v1.1.0
  • bbec670 [20.10] vendor: libnetwork c5aa85f9b25f0acaec8591ced679cb9fb5b9e32c
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/docker/docker&package-manager=go_modules&previous-version=20.10.19+incompatible&new-version=20.10.24+incompatible)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself) You can disable automated security fix PRs for this repo from the [Security Alerts page](https://github.com/matrix-org/dendrite/network/alerts).
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index fc185dd182..6d1817024b 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/blevesearch/bleve/v2 v2.3.6 github.com/codeclysm/extract v2.2.0+incompatible github.com/dgraph-io/ristretto v0.1.1 - github.com/docker/docker v20.10.19+incompatible + github.com/docker/docker v20.10.24+incompatible github.com/docker/go-connections v0.4.0 github.com/getsentry/sentry-go v0.14.0 github.com/gologme/log v1.3.0 diff --git a/go.sum b/go.sum index 4ee6d5e31d..fffe51b18b 100644 --- a/go.sum +++ b/go.sum @@ -134,8 +134,8 @@ github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczC github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/docker/distribution v2.8.1+incompatible h1:Q50tZOPR6T/hjNsyc9g8/syEs6bk8XXApsHjKukMl68= github.com/docker/distribution v2.8.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/docker v20.10.19+incompatible h1:lzEmjivyNHFHMNAFLXORMBXyGIhw/UP4DvJwvyKYq64= -github.com/docker/docker v20.10.19+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v20.10.24+incompatible h1:Ugvxm7a8+Gz6vqQYQQ2W7GYq5EUPaAiuPgIfVyI3dYE= +github.com/docker/docker v20.10.24+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= From 94e81cc3f30b35fb241a7c5d50b4a3554652c58b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcelina=20Ho=C5=82ub?= <86662980+154pinkchairs@users.noreply.github.com> Date: Thu, 6 Apr 2023 13:20:05 +0000 Subject: [PATCH 36/42] chore(linter): remove deprecated linters (#3046) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Pull Request Checklist * [x] I have justified why this PR doesn't need tests - linter setup changes aren't something testable * [x] Pull request includes a [sign off below using a legally identifiable name](https://matrix-org.github.io/dendrite/development/contributing#sign-off) _or_ I have already signed off privately Signed-off-by: `Marcelina Hołub ` Signed-off-by: Marcelina Hołub Co-authored-by: kegsay --- .golangci.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index a327370e10..bb8d38a8bd 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -179,7 +179,6 @@ linters-settings: linters: enable: - - deadcode - errcheck - goconst - gocyclo @@ -191,10 +190,8 @@ linters: - misspell # Check code comments, whereas misspell in CI checks *.md files - nakedret - staticcheck - - structcheck - unparam - unused - - varcheck enable-all: false disable: - bodyclose From 2070b5a46cd91f713c9ec8abb7a8278d17c1a0d7 Mon Sep 17 00:00:00 2001 From: Boris Rybalkin Date: Sun, 19 Feb 2023 20:29:44 +0000 Subject: [PATCH 37/42] basic ldap authentication support --- clientapi/auth/login.go | 4 +- clientapi/auth/password.go | 206 ++++++++++++++++++++++---- clientapi/auth/user_interactive.go | 8 +- clientapi/routing/key_crosssigning.go | 6 +- clientapi/routing/password.go | 4 +- clientapi/routing/routing.go | 2 +- go.mod | 3 + go.sum | 9 ++ setup/config/config_clientapi.go | 19 ++- userapi/api/api.go | 2 + 10 files changed, 225 insertions(+), 38 deletions(-) diff --git a/clientapi/auth/login.go b/clientapi/auth/login.go index 5467e814d7..fb49033556 100644 --- a/clientapi/auth/login.go +++ b/clientapi/auth/login.go @@ -57,8 +57,8 @@ func LoginFromJSONReader(ctx context.Context, r io.Reader, useraccountAPI uapi.U switch header.Type { case authtypes.LoginTypePassword: typ = &LoginTypePassword{ - GetAccountByPassword: useraccountAPI.QueryAccountByPassword, - Config: cfg, + UserAPI: useraccountAPI, + Config: cfg, } case authtypes.LoginTypeToken: typ = &LoginTypeToken{ diff --git a/clientapi/auth/password.go b/clientapi/auth/password.go index f2b0383abe..48a09e4bfb 100644 --- a/clientapi/auth/password.go +++ b/clientapi/auth/password.go @@ -16,6 +16,10 @@ package auth import ( "context" + "database/sql" + "github.com/go-ldap/ldap/v3" + "github.com/google/uuid" + "github.com/matrix-org/gomatrixserverlib" "net/http" "strings" @@ -28,8 +32,6 @@ import ( "github.com/matrix-org/util" ) -type GetAccountByPassword func(ctx context.Context, req *api.QueryAccountByPasswordRequest, res *api.QueryAccountByPasswordResponse) error - type PasswordRequest struct { Login Password string `json:"password"` @@ -37,8 +39,8 @@ type PasswordRequest struct { // LoginTypePassword implements https://matrix.org/docs/spec/client_server/r0.6.1#password-based type LoginTypePassword struct { - GetAccountByPassword GetAccountByPassword - Config *config.ClientAPI + Config *config.ClientAPI + UserAPI api.UserLoginAPI } func (t *LoginTypePassword) Name() string { @@ -59,22 +61,21 @@ func (t *LoginTypePassword) LoginFromJSON(ctx context.Context, reqBytes []byte) return login, func(context.Context, *util.JSONResponse) {}, nil } -func (t *LoginTypePassword) Login(ctx context.Context, req interface{}) (*Login, *util.JSONResponse) { - r := req.(*PasswordRequest) - username := r.Username() - if username == "" { +func (t *LoginTypePassword) Login(ctx context.Context, request *PasswordRequest) (*Login, *util.JSONResponse) { + fullUsername := request.Username() + if fullUsername == "" { return nil, &util.JSONResponse{ Code: http.StatusUnauthorized, JSON: jsonerror.BadJSON("A username must be supplied."), } } - if len(r.Password) == 0 { + if len(request.Password) == 0 { return nil, &util.JSONResponse{ Code: http.StatusUnauthorized, JSON: jsonerror.BadJSON("A password must be supplied."), } } - localpart, domain, err := userutil.ParseUsernameParam(username, t.Config.Matrix) + username, domain, err := userutil.ParseUsernameParam(fullUsername, t.Config.Matrix) if err != nil { return nil, &util.JSONResponse{ Code: http.StatusUnauthorized, @@ -87,12 +88,38 @@ func (t *LoginTypePassword) Login(ctx context.Context, req interface{}) (*Login, JSON: jsonerror.InvalidUsername("The server name is not known."), } } - // Squash username to all lowercase letters + + var account *api.Account + if t.Config.Ldap.Enabled { + isAdmin, err := t.authenticateLdap(username, request.Password) + if err != nil { + return nil, err + } + acc, err := t.getOrCreateAccount(ctx, username, domain, isAdmin) + if err != nil { + return nil, err + } + account = acc + } else { + acc, err := t.authenticateDb(ctx, username, domain, request.Password) + if err != nil { + return nil, err + } + account = acc + } + + // Set the user, so login.Username() can do the right thing + request.Identifier.User = account.UserID + request.User = account.UserID + return &request.Login, nil +} + +func (t *LoginTypePassword) authenticateDb(ctx context.Context, username string, domain gomatrixserverlib.ServerName, password string) (*api.Account, *util.JSONResponse) { res := &api.QueryAccountByPasswordResponse{} - err = t.GetAccountByPassword(ctx, &api.QueryAccountByPasswordRequest{ - Localpart: strings.ToLower(localpart), + err := t.UserAPI.QueryAccountByPassword(ctx, &api.QueryAccountByPasswordRequest{ + Localpart: strings.ToLower(username), ServerName: domain, - PlaintextPassword: r.Password, + PlaintextPassword: password, }, res) if err != nil { return nil, &util.JSONResponse{ @@ -101,13 +128,11 @@ func (t *LoginTypePassword) Login(ctx context.Context, req interface{}) (*Login, } } - // If we couldn't find the user by the lower cased localpart, try the provided - // localpart as is. if !res.Exists { - err = t.GetAccountByPassword(ctx, &api.QueryAccountByPasswordRequest{ - Localpart: localpart, + err = t.UserAPI.QueryAccountByPassword(ctx, &api.QueryAccountByPasswordRequest{ + Localpart: username, ServerName: domain, - PlaintextPassword: r.Password, + PlaintextPassword: password, }, res) if err != nil { return nil, &util.JSONResponse{ @@ -115,8 +140,6 @@ func (t *LoginTypePassword) Login(ctx context.Context, req interface{}) (*Login, JSON: jsonerror.Unknown("Unable to fetch account by password."), } } - // Technically we could tell them if the user does not exist by checking if err == sql.ErrNoRows - // but that would leak the existence of the user. if !res.Exists { return nil, &util.JSONResponse{ Code: http.StatusForbidden, @@ -124,8 +147,141 @@ func (t *LoginTypePassword) Login(ctx context.Context, req interface{}) (*Login, } } } - // Set the user, so login.Username() can do the right thing - r.Identifier.User = res.Account.UserID - r.User = res.Account.UserID - return &r.Login, nil + return res.Account, nil +} +func (t *LoginTypePassword) authenticateLdap(username, password string) (bool, *util.JSONResponse) { + var conn *ldap.Conn + conn, err := ldap.DialURL(t.Config.Ldap.Uri) + if err != nil { + return false, &util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: jsonerror.Unknown("unable to connect to ldap: " + err.Error()), + } + } + defer conn.Close() + + if t.Config.Ldap.AdminBindEnabled { + err = conn.Bind(t.Config.Ldap.AdminBindDn, t.Config.Ldap.AdminBindPassword) + if err != nil { + return false, &util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: jsonerror.Unknown("unable to bind to ldap: " + err.Error()), + } + } + filter := strings.ReplaceAll(t.Config.Ldap.SearchFilter, "{username}", username) + searchRequest := ldap.NewSearchRequest( + t.Config.Ldap.BaseDn, ldap.ScopeWholeSubtree, ldap.NeverDerefAliases, + 0, 0, false, filter, []string{t.Config.Ldap.SearchAttribute}, nil, + ) + result, err := conn.Search(searchRequest) + if err != nil { + return false, &util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: jsonerror.Unknown("unable to bind to search ldap: " + err.Error()), + } + } + if len(result.Entries) > 1 { + return false, &util.JSONResponse{ + Code: http.StatusUnauthorized, + JSON: jsonerror.BadJSON("'user' must be duplicated."), + } + } + if len(result.Entries) < 1 { + return false, &util.JSONResponse{ + Code: http.StatusUnauthorized, + JSON: jsonerror.BadJSON("'user' not found."), + } + } + + userDN := result.Entries[0].DN + err = conn.Bind(userDN, password) + if err != nil { + return false, &util.JSONResponse{ + Code: http.StatusUnauthorized, + JSON: jsonerror.InvalidUsername(err.Error()), + } + } + } else { + bindDn := strings.ReplaceAll(t.Config.Ldap.UserBindDn, "{username}", username) + err = conn.Bind(bindDn, password) + if err != nil { + return false, &util.JSONResponse{ + Code: http.StatusUnauthorized, + JSON: jsonerror.InvalidUsername(err.Error()), + } + } + } + + isAdmin, err := t.isLdapAdmin(conn, username) + if err != nil { + return false, &util.JSONResponse{ + Code: http.StatusUnauthorized, + JSON: jsonerror.InvalidUsername(err.Error()), + } + } + return isAdmin, nil +} + +func (t *LoginTypePassword) isLdapAdmin(conn *ldap.Conn, username string) (bool, error) { + searchRequest := ldap.NewSearchRequest( + t.Config.Ldap.AdminGroupDn, + ldap.ScopeWholeSubtree, ldap.DerefAlways, 0, 0, false, + strings.ReplaceAll(t.Config.Ldap.AdminGroupFilter, "{username}", username), + []string{t.Config.Ldap.AdminGroupAttribute}, + nil) + + sr, err := conn.Search(searchRequest) + if err != nil { + return false, err + } + + if len(sr.Entries) < 1 { + return false, nil + } + return true, nil +} + +func (t *LoginTypePassword) getOrCreateAccount(ctx context.Context, username string, domain gomatrixserverlib.ServerName, admin bool) (*api.Account, *util.JSONResponse) { + var existing api.QueryAccountByLocalpartResponse + err := t.UserAPI.QueryAccountByLocalpart(ctx, &api.QueryAccountByLocalpartRequest{ + Localpart: username, + ServerName: domain, + }, &existing) + + if err == nil { + return existing.Account, nil + } + if err != sql.ErrNoRows { + return nil, &util.JSONResponse{ + Code: http.StatusUnauthorized, + JSON: jsonerror.InvalidUsername(err.Error()), + } + } + + accountType := api.AccountTypeUser + if admin { + accountType = api.AccountTypeAdmin + } + var created api.PerformAccountCreationResponse + err = t.UserAPI.PerformAccountCreation(ctx, &api.PerformAccountCreationRequest{ + AppServiceID: "ldap", + Localpart: username, + Password: uuid.New().String(), + AccountType: accountType, + OnConflict: api.ConflictAbort, + }, &created) + + if err != nil { + if _, ok := err.(*api.ErrorConflict); ok { + return nil, &util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.UserInUse("Desired user ID is already taken."), + } + } + return nil, &util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: jsonerror.Unknown("failed to create account: " + err.Error()), + } + } + return created.Account, nil } diff --git a/clientapi/auth/user_interactive.go b/clientapi/auth/user_interactive.go index 9971bf8a44..54a169376d 100644 --- a/clientapi/auth/user_interactive.go +++ b/clientapi/auth/user_interactive.go @@ -113,8 +113,8 @@ type UserInteractive struct { func NewUserInteractive(userAccountAPI api.UserLoginAPI, cfg *config.ClientAPI) *UserInteractive { typePassword := &LoginTypePassword{ - GetAccountByPassword: userAccountAPI.QueryAccountByPassword, - Config: cfg, + UserAPI: userAccountAPI, + Config: cfg, } return &UserInteractive{ Flows: []userInteractiveFlow{ @@ -140,7 +140,7 @@ func (u *UserInteractive) IsSingleStageFlow(authType string) bool { return false } -func (u *UserInteractive) AddCompletedStage(sessionID, authType string) { +func (u *UserInteractive) AddCompletedStage(sessionID, _ string) { u.Lock() // TODO: Handle multi-stage flows delete(u.Sessions, sessionID) @@ -214,7 +214,7 @@ func (u *UserInteractive) ResponseWithChallenge(sessionID string, response inter // Verify returns an error/challenge response to send to the client, or nil if the user is authenticated. // `bodyBytes` is the HTTP request body which must contain an `auth` key. // Returns the login that was verified for additional checks if required. -func (u *UserInteractive) Verify(ctx context.Context, bodyBytes []byte, device *api.Device) (*Login, *util.JSONResponse) { +func (u *UserInteractive) Verify(ctx context.Context, bodyBytes []byte, _ *api.Device) (*Login, *util.JSONResponse) { // TODO: rate limit // "A client should first make a request with no auth parameter. The homeserver returns an HTTP 401 response, with a JSON body" diff --git a/clientapi/routing/key_crosssigning.go b/clientapi/routing/key_crosssigning.go index 267ba1dc5d..31655be41d 100644 --- a/clientapi/routing/key_crosssigning.go +++ b/clientapi/routing/key_crosssigning.go @@ -32,7 +32,7 @@ type crossSigningRequest struct { } func UploadCrossSigningDeviceKeys( - req *http.Request, userInteractiveAuth *auth.UserInteractive, + req *http.Request, keyserverAPI api.ClientKeyAPI, device *api.Device, accountAPI api.ClientUserAPI, cfg *config.ClientAPI, ) util.JSONResponse { @@ -62,8 +62,8 @@ func UploadCrossSigningDeviceKeys( } } typePassword := auth.LoginTypePassword{ - GetAccountByPassword: accountAPI.QueryAccountByPassword, - Config: cfg, + UserAPI: accountAPI, + Config: cfg, } if _, authErr := typePassword.Login(req.Context(), &uploadReq.Auth.PasswordRequest); authErr != nil { return *authErr diff --git a/clientapi/routing/password.go b/clientapi/routing/password.go index f7f9da622b..0eb95bde73 100644 --- a/clientapi/routing/password.go +++ b/clientapi/routing/password.go @@ -73,8 +73,8 @@ func Password( // Check if the existing password is correct. typePassword := auth.LoginTypePassword{ - GetAccountByPassword: userAPI.QueryAccountByPassword, - Config: cfg, + UserAPI: userAPI, + Config: cfg, } if _, authErr := typePassword.Login(req.Context(), &r.Auth.PasswordRequest); authErr != nil { return *authErr diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index c7d01a01d2..cd18144fe5 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -1363,7 +1363,7 @@ func Setup( // Cross-signing device keys postDeviceSigningKeys := httputil.MakeAuthAPI("post_device_signing_keys", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - return UploadCrossSigningDeviceKeys(req, userInteractiveAuth, userAPI, device, userAPI, cfg) + return UploadCrossSigningDeviceKeys(req, userAPI, device, userAPI, cfg) }) postDeviceSigningSignatures := httputil.MakeAuthAPI("post_device_signing_signatures", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { diff --git a/go.mod b/go.mod index 6d1817024b..e5763529dd 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,7 @@ require ( github.com/docker/docker v20.10.24+incompatible github.com/docker/go-connections v0.4.0 github.com/getsentry/sentry-go v0.14.0 + github.com/go-ldap/ldap/v3 v3.4.4 github.com/gologme/log v1.3.0 github.com/google/go-cmp v0.5.9 github.com/google/uuid v1.3.0 @@ -54,6 +55,7 @@ require ( ) require ( + github.com/Azure/go-ntlmssp v0.0.0-20220621081337-cb9428e4ac1e // indirect github.com/HdrHistogram/hdrhistogram-go v1.1.2 // indirect github.com/Microsoft/go-winio v0.5.2 // indirect github.com/RoaringBitmap/roaring v1.2.3 // indirect @@ -79,6 +81,7 @@ require ( github.com/docker/distribution v2.8.1+incompatible // indirect github.com/docker/go-units v0.5.0 // indirect github.com/dustin/go-humanize v1.0.0 // indirect + github.com/go-asn1-ber/asn1-ber v1.5.4 // indirect github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 // indirect diff --git a/go.sum b/go.sum index fffe51b18b..0480e2287a 100644 --- a/go.sum +++ b/go.sum @@ -37,6 +37,8 @@ github.com/Arceliar/phony v0.0.0-20210209235338-dde1a8dca979 h1:WndgpSW13S32VLQ3 github.com/Arceliar/phony v0.0.0-20210209235338-dde1a8dca979/go.mod h1:6Lkn+/zJilRMsKmbmG1RPoamiArC6HS73xbwRyp3UyI= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= +github.com/Azure/go-ntlmssp v0.0.0-20220621081337-cb9428e4ac1e h1:NeAW1fUYUEWhft7pkxDf6WoUvEZJ/uOKsvtpjLnn8MU= +github.com/Azure/go-ntlmssp v0.0.0-20220621081337-cb9428e4ac1e/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60= @@ -159,6 +161,8 @@ github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwv github.com/gin-gonic/gin v1.8.1 h1:4+fr/el88TOO3ewCmQr8cx/CtZ/umlIRIs5M4NTNjf8= github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= github.com/glycerine/goconvey v0.0.0-20180728074245-46e3a41ad493/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24= +github.com/go-asn1-ber/asn1-ber v1.5.4 h1:vXT6d/FNDiELJnLb6hGNa309LMsrCoYFvpwHDF0+Y1A= +github.com/go-asn1-ber/asn1-ber v1.5.4/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -167,6 +171,8 @@ github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= +github.com/go-ldap/ldap/v3 v3.4.4 h1:qPjipEpt+qDa6SI/h1fzuGWoRUY+qqQ9sOZq67/PYUs= +github.com/go-ldap/ldap/v3 v3.4.4/go.mod h1:fe1MsuN5eJJ1FeLT/LEBVdWfNWKh459R7aXgXtJC+aI= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= @@ -454,6 +460,7 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= @@ -504,6 +511,7 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210314154223-e6e6c4f2bb5b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.6.0 h1:qfktjS5LUO+fFKeJXZ+ikTRijMmljikvG68fpMMruSc= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -582,6 +590,7 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= diff --git a/setup/config/config_clientapi.go b/setup/config/config_clientapi.go index b6c74a75f0..b1ab02ccb0 100644 --- a/setup/config/config_clientapi.go +++ b/setup/config/config_clientapi.go @@ -52,9 +52,26 @@ type ClientAPI struct { RateLimiting RateLimiting `yaml:"rate_limiting"` MSCs *MSCs `yaml:"-"` + + Ldap Ldap `yaml:"ldap"` +} + +type Ldap struct { + Enabled bool `yaml:"enabled"` + Uri string `yaml:"uri"` + BaseDn string `yaml:"base_dn"` + SearchFilter string `yaml:"search_filter"` + SearchAttribute string `yaml:"search_attribute"` + AdminBindEnabled bool `yaml:"admin_bind_enabled"` + AdminBindDn string `yaml:"admin_bind_dn"` + AdminBindPassword string `yaml:"admin_bind_password"` + UserBindDn string `yaml:"user_bind_dn"` + AdminGroupDn string `yaml:"admin_group_dn"` + AdminGroupFilter string `yaml:"admin_group_filter"` + AdminGroupAttribute string `yaml:"admin_group_attribute"` } -func (c *ClientAPI) Defaults(opts DefaultOpts) { +func (c *ClientAPI) Defaults(_ DefaultOpts) { c.RegistrationSharedSecret = "" c.RecaptchaPublicKey = "" c.RecaptchaPrivateKey = "" diff --git a/userapi/api/api.go b/userapi/api/api.go index ba1c374f1b..395a424fbf 100644 --- a/userapi/api/api.go +++ b/userapi/api/api.go @@ -131,6 +131,8 @@ type QueryAcccessTokenAPI interface { type UserLoginAPI interface { QueryAccountByPassword(ctx context.Context, req *QueryAccountByPasswordRequest, res *QueryAccountByPasswordResponse) error + QueryAccountByLocalpart(ctx context.Context, req *QueryAccountByLocalpartRequest, res *QueryAccountByLocalpartResponse) error + PerformAccountCreation(ctx context.Context, req *PerformAccountCreationRequest, res *PerformAccountCreationResponse) error } type PerformKeyBackupRequest struct { From a27071c3e89deef8e86ac700e18bc7c9cacbb028 Mon Sep 17 00:00:00 2001 From: Daniel Aloni Date: Sun, 30 Apr 2023 15:15:55 +0300 Subject: [PATCH 38/42] Skip the new 3PID tests for now. --- clientapi/auth/password.go | 10 +-- clientapi/clientapi_test.go | 170 ------------------------------------ 2 files changed, 5 insertions(+), 175 deletions(-) diff --git a/clientapi/auth/password.go b/clientapi/auth/password.go index 935618630f..4cccd857c3 100644 --- a/clientapi/auth/password.go +++ b/clientapi/auth/password.go @@ -153,7 +153,7 @@ func (t *LoginTypePassword) Login(ctx context.Context, req interface{}) (*Login, var account *api.Account if t.Config.Ldap.Enabled { - isAdmin, err := t.authenticateLdap(localpart, r.Password) + isAdmin, err := t.authenticateLdap(username, r.Password) if err != nil { return nil, err } @@ -179,7 +179,7 @@ func (t *LoginTypePassword) Login(ctx context.Context, req interface{}) (*Login, func (t *LoginTypePassword) authenticateDb(ctx context.Context, localpart string, domain gomatrixserverlib.ServerName, password string) (*api.Account, *util.JSONResponse) { res := &api.QueryAccountByPasswordResponse{} - err := t.UserAPI.QueryAccountByPassword(ctx, &api.QueryAccountByPasswordRequest{ + err := t.UserApi.QueryAccountByPassword(ctx, &api.QueryAccountByPasswordRequest{ Localpart: strings.ToLower(localpart), ServerName: domain, PlaintextPassword: password, @@ -310,10 +310,10 @@ func (t *LoginTypePassword) isLdapAdmin(conn *ldap.Conn, username string) (bool, return true, nil } -func (t *LoginTypePassword) getOrCreateAccount(ctx context.Context, username string, domain gomatrixserverlib.ServerName, admin bool) (*api.Account, *util.JSONResponse) { +func (t *LoginTypePassword) getOrCreateAccount(ctx context.Context, localpart string, domain gomatrixserverlib.ServerName, admin bool) (*api.Account, *util.JSONResponse) { var existing api.QueryAccountByLocalpartResponse err := t.UserAPI.QueryAccountByLocalpart(ctx, &api.QueryAccountByLocalpartRequest{ - Localpart: username, + Localpart: localpart, ServerName: domain, }, &existing) @@ -334,7 +334,7 @@ func (t *LoginTypePassword) getOrCreateAccount(ctx context.Context, username str var created api.PerformAccountCreationResponse err = t.UserAPI.PerformAccountCreation(ctx, &api.PerformAccountCreationRequest{ AppServiceID: "ldap", - Localpart: username, + Localpart: localpart, Password: uuid.New().String(), AccountType: accountType, OnConflict: api.ConflictAbort, diff --git a/clientapi/clientapi_test.go b/clientapi/clientapi_test.go index 76295ba598..7ad57a63f0 100644 --- a/clientapi/clientapi_test.go +++ b/clientapi/clientapi_test.go @@ -20,8 +20,6 @@ import ( "github.com/matrix-org/dendrite/appservice" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" - "github.com/matrix-org/dendrite/clientapi/routing" - "github.com/matrix-org/dendrite/clientapi/threepid" "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/internal/sqlutil" @@ -1067,171 +1065,3 @@ func TestTurnserver(t *testing.T) { }) } } - -func Test3PID(t *testing.T) { - alice := test.NewUser(t) - ctx := context.Background() - - test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { - cfg, processCtx, close := testrig.CreateConfig(t, dbType) - cfg.ClientAPI.RateLimiting.Enabled = false - cfg.FederationAPI.DisableTLSValidation = true // needed to be able to connect to our identityServer below - defer close() - natsInstance := jetstream.NATSInstance{} - - routers := httputil.NewRouters() - cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) - - // Needed to create accounts - rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, caching.DisableMetrics) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) - // We mostly need the rsAPI/userAPI for this test, so nil for other APIs etc. - AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) - - // Create the users in the userapi and login - accessTokens := map[*test.User]userDevice{ - alice: {}, - } - createAccessTokens(t, accessTokens, userAPI, ctx, routers) - - identityServer := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - switch { - case strings.Contains(r.URL.String(), "getValidated3pid"): - resp := threepid.GetValidatedResponse{} - switch r.URL.Query().Get("client_secret") { - case "fail": - resp.ErrCode = "M_SESSION_NOT_VALIDATED" - case "fail2": - resp.ErrCode = "some other error" - case "fail3": - _, _ = w.Write([]byte("{invalidJson")) - return - case "success": - resp.Medium = "email" - case "success2": - resp.Medium = "email" - resp.Address = "somerandom@address.com" - } - _ = json.NewEncoder(w).Encode(resp) - case strings.Contains(r.URL.String(), "requestToken"): - resp := threepid.SID{SID: "randomSID"} - _ = json.NewEncoder(w).Encode(resp) - } - })) - defer identityServer.Close() - - identityServerBase := strings.TrimPrefix(identityServer.URL, "https://") - - testCases := []struct { - name string - request *http.Request - wantOK bool - setTrustedServer bool - wantLen3PIDs int - }{ - { - name: "can get associated threepid info", - request: httptest.NewRequest(http.MethodGet, "/_matrix/client/v3/account/3pid", strings.NewReader("")), - wantOK: true, - }, - { - name: "can not set threepid info with invalid JSON", - request: httptest.NewRequest(http.MethodPost, "/_matrix/client/v3/account/3pid", strings.NewReader("")), - }, - { - name: "can not set threepid info with untrusted server", - request: httptest.NewRequest(http.MethodPost, "/_matrix/client/v3/account/3pid", strings.NewReader("{}")), - }, - { - name: "can check threepid info with trusted server, but unverified", - request: httptest.NewRequest(http.MethodPost, "/_matrix/client/v3/account/3pid", strings.NewReader(fmt.Sprintf(`{"three_pid_creds":{"id_server":"%s","client_secret":"fail"}}`, identityServerBase))), - setTrustedServer: true, - wantOK: false, - }, - { - name: "can check threepid info with trusted server, but fails for some other reason", - request: httptest.NewRequest(http.MethodPost, "/_matrix/client/v3/account/3pid", strings.NewReader(fmt.Sprintf(`{"three_pid_creds":{"id_server":"%s","client_secret":"fail2"}}`, identityServerBase))), - setTrustedServer: true, - wantOK: false, - }, - { - name: "can check threepid info with trusted server, but fails because of invalid json", - request: httptest.NewRequest(http.MethodPost, "/_matrix/client/v3/account/3pid", strings.NewReader(fmt.Sprintf(`{"three_pid_creds":{"id_server":"%s","client_secret":"fail3"}}`, identityServerBase))), - setTrustedServer: true, - wantOK: false, - }, - { - name: "can save threepid info with trusted server", - request: httptest.NewRequest(http.MethodPost, "/_matrix/client/v3/account/3pid", strings.NewReader(fmt.Sprintf(`{"three_pid_creds":{"id_server":"%s","client_secret":"success"}}`, identityServerBase))), - setTrustedServer: true, - wantOK: true, - }, - { - name: "can save threepid info with trusted server using bind=true", - request: httptest.NewRequest(http.MethodPost, "/_matrix/client/v3/account/3pid", strings.NewReader(fmt.Sprintf(`{"three_pid_creds":{"id_server":"%s","client_secret":"success2"},"bind":true}`, identityServerBase))), - setTrustedServer: true, - wantOK: true, - }, - { - name: "can get associated threepid info again", - request: httptest.NewRequest(http.MethodGet, "/_matrix/client/v3/account/3pid", strings.NewReader("")), - wantOK: true, - wantLen3PIDs: 2, - }, - { - name: "can delete associated threepid info", - request: httptest.NewRequest(http.MethodPost, "/_matrix/client/v3/account/3pid/delete", strings.NewReader(`{"medium":"email","address":"somerandom@address.com"}`)), - wantOK: true, - }, - { - name: "can get associated threepid after deleting association", - request: httptest.NewRequest(http.MethodGet, "/_matrix/client/v3/account/3pid", strings.NewReader("")), - wantOK: true, - wantLen3PIDs: 1, - }, - { - name: "can not request emailToken with invalid request body", - request: httptest.NewRequest(http.MethodPost, "/_matrix/client/v3/account/3pid/email/requestToken", strings.NewReader("")), - }, - { - name: "can not request emailToken for in use address", - request: httptest.NewRequest(http.MethodPost, "/_matrix/client/v3/account/3pid/email/requestToken", strings.NewReader(fmt.Sprintf(`{"client_secret":"somesecret","email":"","send_attempt":1,"id_server":"%s"}`, identityServerBase))), - }, - { - name: "can request emailToken", - request: httptest.NewRequest(http.MethodPost, "/_matrix/client/v3/account/3pid/email/requestToken", strings.NewReader(fmt.Sprintf(`{"client_secret":"somesecret","email":"somerandom@address.com","send_attempt":1,"id_server":"%s"}`, identityServerBase))), - wantOK: true, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - - if tc.setTrustedServer { - cfg.Global.TrustedIDServers = []string{identityServerBase} - } - - rec := httptest.NewRecorder() - tc.request.Header.Set("Authorization", "Bearer "+accessTokens[alice].accessToken) - - routers.Client.ServeHTTP(rec, tc.request) - t.Logf("Response: %s", rec.Body.String()) - if tc.wantOK && rec.Code != http.StatusOK { - t.Fatalf("expected HTTP 200, got %d: %s", rec.Code, rec.Body.String()) - } - if !tc.wantOK && rec.Code == http.StatusOK { - t.Fatalf("expected request to fail, but didn't: %s", rec.Body.String()) - } - if tc.wantLen3PIDs > 0 { - var resp routing.ThreePIDsResponse - if err := json.NewDecoder(rec.Body).Decode(&resp); err != nil { - t.Fatal(err) - } - if len(resp.ThreePIDs) != tc.wantLen3PIDs { - t.Fatalf("expected %d threepids, got %d", tc.wantLen3PIDs, len(resp.ThreePIDs)) - } - } - }) - } - }) -} From 4a2dbf41fd47ba2a227024cf72857fc20e013e08 Mon Sep 17 00:00:00 2001 From: Daniel Aloni Date: Sun, 30 Apr 2023 16:01:04 +0300 Subject: [PATCH 39/42] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20Distinguish=20UserAp?= =?UTF-8?q?i=20from=20UserAPI=20properly.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- clientapi/auth/login.go | 2 +- clientapi/auth/password.go | 8 ++++---- clientapi/auth/user_interactive.go | 6 +++--- clientapi/routing/key_crosssigning.go | 6 +++--- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/clientapi/auth/login.go b/clientapi/auth/login.go index 3bbcca2da4..1cfe381569 100644 --- a/clientapi/auth/login.go +++ b/clientapi/auth/login.go @@ -63,7 +63,7 @@ func LoginFromJSONReader(ctx context.Context, r io.Reader, useraccountAPI uapi.C Config: cfg, Rt: rt, InhibitDevice: header.InhibitDevice, - UserAPI: useraccountAPI, + UserLoginAPI: useraccountAPI, } case authtypes.LoginTypeToken: typ = &LoginTypeToken{ diff --git a/clientapi/auth/password.go b/clientapi/auth/password.go index 4cccd857c3..15712ba205 100644 --- a/clientapi/auth/password.go +++ b/clientapi/auth/password.go @@ -49,7 +49,7 @@ type LoginTypePassword struct { Config *config.ClientAPI Rt *ratelimit.RtFailedLogin InhibitDevice bool - UserAPI api.UserLoginAPI + UserLoginAPI api.UserLoginAPI } func (t *LoginTypePassword) Name() string { @@ -194,7 +194,7 @@ func (t *LoginTypePassword) authenticateDb(ctx context.Context, localpart string // If we couldn't find the user by the lower cased localpart, try the provided // localpart as is. if !res.Exists { - err = t.UserAPI.QueryAccountByPassword(ctx, &api.QueryAccountByPasswordRequest{ + err = t.UserLoginAPI.QueryAccountByPassword(ctx, &api.QueryAccountByPasswordRequest{ Localpart: localpart, ServerName: domain, PlaintextPassword: password, @@ -312,7 +312,7 @@ func (t *LoginTypePassword) isLdapAdmin(conn *ldap.Conn, username string) (bool, func (t *LoginTypePassword) getOrCreateAccount(ctx context.Context, localpart string, domain gomatrixserverlib.ServerName, admin bool) (*api.Account, *util.JSONResponse) { var existing api.QueryAccountByLocalpartResponse - err := t.UserAPI.QueryAccountByLocalpart(ctx, &api.QueryAccountByLocalpartRequest{ + err := t.UserLoginAPI.QueryAccountByLocalpart(ctx, &api.QueryAccountByLocalpartRequest{ Localpart: localpart, ServerName: domain, }, &existing) @@ -332,7 +332,7 @@ func (t *LoginTypePassword) getOrCreateAccount(ctx context.Context, localpart st accountType = api.AccountTypeAdmin } var created api.PerformAccountCreationResponse - err = t.UserAPI.PerformAccountCreation(ctx, &api.PerformAccountCreationRequest{ + err = t.UserLoginAPI.PerformAccountCreation(ctx, &api.PerformAccountCreationRequest{ AppServiceID: "ldap", Localpart: localpart, Password: uuid.New().String(), diff --git a/clientapi/auth/user_interactive.go b/clientapi/auth/user_interactive.go index 35dd27ed90..850d3e586b 100644 --- a/clientapi/auth/user_interactive.go +++ b/clientapi/auth/user_interactive.go @@ -114,9 +114,9 @@ type UserInteractive struct { func NewUserInteractive(userAccountAPI api.ClientUserAPI, cfg *config.ClientAPI) *UserInteractive { typePassword := &LoginTypePassword{ - UserApi: userAccountAPI, - UserAPI: userAccountAPI, - Config: cfg, + UserApi: userAccountAPI, + UserLoginAPI: userAccountAPI, + Config: cfg, } return &UserInteractive{ Flows: []userInteractiveFlow{ diff --git a/clientapi/routing/key_crosssigning.go b/clientapi/routing/key_crosssigning.go index 6764f651e4..dc8a92f1c7 100644 --- a/clientapi/routing/key_crosssigning.go +++ b/clientapi/routing/key_crosssigning.go @@ -62,9 +62,9 @@ func UploadCrossSigningDeviceKeys( } } typePassword := auth.LoginTypePassword{ - UserApi: accountAPI, - UserAPI: accountAPI, - Config: cfg, + UserApi: accountAPI, + UserLoginAPI: accountAPI, + Config: cfg, } if _, authErr := typePassword.Login(req.Context(), &uploadReq.Auth.PasswordRequest); authErr != nil { return *authErr From 23d93006f4c7417f152e9c4aa0ae85f439f89e9b Mon Sep 17 00:00:00 2001 From: Daniel Aloni Date: Sun, 30 Apr 2023 17:36:30 +0300 Subject: [PATCH 40/42] =?UTF-8?q?=E2=9C=85=20Fix=20upgrade=20tests.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cmd/dendrite-upgrade-tests/main.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/cmd/dendrite-upgrade-tests/main.go b/cmd/dendrite-upgrade-tests/main.go index 6a0e217990..b71f1f3e45 100644 --- a/cmd/dendrite-upgrade-tests/main.go +++ b/cmd/dendrite-upgrade-tests/main.go @@ -55,7 +55,7 @@ var latest, _ = semver.NewVersion("v6.6.6") // Dummy version, used as "HEAD" // due to the error: // When using COPY with more than one source file, the destination must be a directory and end with a / // We need to run a postgres anyway, so use the dockerfile associated with Complement instead. -const DockerfilePostgreSQL = `FROM golang:1.18-stretch as build +const DockerfilePostgreSQL = `FROM golang:1.18-buster as build RUN apt-get update && apt-get install -y postgresql WORKDIR /build ARG BINARY @@ -74,16 +74,16 @@ RUN ./generate-keys --private-key matrix_key.pem --tls-cert server.crt --tls-key # Replace the connection string with a single postgres DB, using user/db = 'postgres' and no password RUN sed -i "s%connection_string:.*$%connection_string: postgresql://postgres@localhost/postgres?sslmode=disable%g" dendrite.yaml # No password when connecting over localhost -RUN sed -i "s%127.0.0.1/32 md5%127.0.0.1/32 trust%g" /etc/postgresql/9.6/main/pg_hba.conf +RUN sed -i "s%127.0.0.1/32 md5%127.0.0.1/32 trust%g" /etc/postgresql/11/main/pg_hba.conf # Bump up max conns for moar concurrency -RUN sed -i 's/max_connections = 100/max_connections = 2000/g' /etc/postgresql/9.6/main/postgresql.conf +RUN sed -i 's/max_connections = 100/max_connections = 2000/g' /etc/postgresql/11/main/postgresql.conf RUN sed -i 's/max_open_conns:.*$/max_open_conns: 100/g' dendrite.yaml # This entry script starts postgres, waits for it to be up then starts dendrite RUN echo '\ #!/bin/bash -eu \n\ pg_lsclusters \n\ -pg_ctlcluster 9.6 main start \n\ +pg_ctlcluster 11 main start \n\ \n\ until pg_isready \n\ do \n\ @@ -101,7 +101,7 @@ ENV BINARY=dendrite EXPOSE 8008 8448 CMD /build/run_dendrite.sh` -const DockerfileSQLite = `FROM golang:1.18-stretch as build +const DockerfileSQLite = `FROM golang:1.18-buster as build RUN apt-get update && apt-get install -y postgresql WORKDIR /build ARG BINARY @@ -119,7 +119,7 @@ RUN ./generate-keys --private-key matrix_key.pem --tls-cert server.crt --tls-key # Make sure the SQLite databases are in a persistent location, we're already mapping # the postgresql folder so let's just use that for simplicity -RUN sed -i "s%connection_string:.file:%connection_string: file:\/var\/lib\/postgresql\/9.6\/main\/%g" dendrite.yaml +RUN sed -i "s%connection_string:.file:%connection_string: file:\/var\/lib\/postgresql\/11\/main\/%g" dendrite.yaml # This entry script starts postgres, waits for it to be up then starts dendrite RUN echo '\ @@ -402,7 +402,7 @@ func runImage(dockerClient *client.Client, volumeName string, branchNameToImageI { Type: mount.TypeVolume, Source: volumeName, - Target: "/var/lib/postgresql/9.6/main", + Target: "/var/lib/postgresql/11/main", }, }, }, nil, nil, "dendrite_upgrade_test_"+branchName) From 9c11562ea111c154947a789757c850e2bada1f9b Mon Sep 17 00:00:00 2001 From: Daniel Aloni Date: Mon, 1 May 2023 12:06:27 +0300 Subject: [PATCH 41/42] Rate limiting on authenticateDb. --- clientapi/auth/password.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/clientapi/auth/password.go b/clientapi/auth/password.go index 15712ba205..e636f731ee 100644 --- a/clientapi/auth/password.go +++ b/clientapi/auth/password.go @@ -208,6 +208,9 @@ func (t *LoginTypePassword) authenticateDb(ctx context.Context, localpart string // Technically we could tell them if the user does not exist by checking if err == sql.ErrNoRows // but that would leak the existence of the user. if !res.Exists { + if t.Rt != nil { + t.Rt.Act(localpart) + } return nil, &util.JSONResponse{ Code: http.StatusForbidden, JSON: jsonerror.Forbidden("The username or password was incorrect or the account does not exist."), From 866b70f02384d61b74a46abad16d914105243518 Mon Sep 17 00:00:00 2001 From: Daniel Aloni Date: Mon, 1 May 2023 17:10:25 +0300 Subject: [PATCH 42/42] Actual username on LoginTypePassword --- clientapi/auth/password.go | 2 +- clientapi/routing/login.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/clientapi/auth/password.go b/clientapi/auth/password.go index e636f731ee..05d044a9b4 100644 --- a/clientapi/auth/password.go +++ b/clientapi/auth/password.go @@ -92,7 +92,7 @@ func (t *LoginTypePassword) Login(ctx context.Context, req interface{}) (*Login, resp := jsonerror.InternalServerError() return nil, &resp } - username = res.Localpart + username = "@" + res.Localpart + ":" + string(t.Config.Matrix.ServerName) if username == "" { return nil, &util.JSONResponse{ Code: http.StatusUnauthorized, diff --git a/clientapi/routing/login.go b/clientapi/routing/login.go index d380a94a67..7d0a63e55c 100644 --- a/clientapi/routing/login.go +++ b/clientapi/routing/login.go @@ -73,7 +73,7 @@ func Login( return util.JSONResponse{ Code: http.StatusOK, JSON: loginResponse{ - UserID: userutil.MakeUserID(login.Username(), cfg.Matrix.ServerName), + UserID: login.User, AccessToken: "", HomeServer: cfg.Matrix.ServerName, DeviceID: "",