From 6940728ffa28cc06fb3efd2e8d3221a3661fa50e Mon Sep 17 00:00:00 2001 From: Daniel Aloni Date: Sun, 9 Jul 2023 15:57:13 +0300 Subject: [PATCH 1/3] =?UTF-8?q?=E2=9C=A8=20/location=5Fsync=20endpoint=20f?= =?UTF-8?q?or=20fetching=20recent=20locations=20in=20a=20room.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/dendrite.yml | 2 +- syncapi/routing/location_sync.go | 64 +++++++++++++++++++ syncapi/routing/routing.go | 10 +++ .../storage/postgres/multiroomcast_table.go | 15 ++++- syncapi/storage/shared/storage_sync.go | 5 ++ syncapi/storage/tables/interface.go | 1 + 6 files changed, 95 insertions(+), 2 deletions(-) create mode 100644 syncapi/routing/location_sync.go diff --git a/.github/workflows/dendrite.yml b/.github/workflows/dendrite.yml index da24d7bc69..13fc44966a 100644 --- a/.github/workflows/dendrite.yml +++ b/.github/workflows/dendrite.yml @@ -76,7 +76,7 @@ jobs: # run go test with go 1.19 test: - timeout-minutes: 10 + timeout-minutes: 15 name: Unit tests runs-on: ubuntu-latest # Service containers to run with `container-job` diff --git a/syncapi/routing/location_sync.go b/syncapi/routing/location_sync.go new file mode 100644 index 0000000000..340726525b --- /dev/null +++ b/syncapi/routing/location_sync.go @@ -0,0 +1,64 @@ +// Copyright 2017 Vector Creations Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package routing + +import ( + "context" + "database/sql" + "net/http" + + "github.com/matrix-org/dendrite/clientapi/jsonerror" + "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/syncapi/storage" + "github.com/matrix-org/dendrite/syncapi/types" + userapi "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/util" + "github.com/sirupsen/logrus" +) + +// https://matrix.org/docs/spec/client_server/r0.6.0#get-matrix-client-r0-rooms-roomid-joined-members +type getLocationSyncResponse struct { + RecentLocations types.MultiRoom `json:"recent_locations"` +} + +// GetLocationSync return each VISIBLE user's most recent location update in the room. +// This is used for the case of newly joined users which require catching up on location events of users, +// but aren't able to retrieve certain location events from /sync, since they joined after them. +func GetLocationSync( + req *http.Request, device *userapi.Device, roomID string, + syncDB storage.Database, rsAPI api.SyncRoomserverAPI, +) util.JSONResponse { + snapshot, err := syncDB.NewDatabaseSnapshot(req.Context()) + if err != nil { + logrus.WithError(err).Error("Failed to get snapshot for locations sync") + return jsonerror.InternalServerError() + } + id, err := snapshot.SelectMaxMultiRoomDataEventId(context.Background()) + if err != nil { + util.GetLogger(req.Context()).WithError(err).Error("failed to get max multiroom data event id") + return jsonerror.InternalServerError() + } + mr, err := snapshot.SelectMultiRoomData(req.Context(), &types.Range{From: 0, To: id}, []string{roomID}) + if err != nil { + if err != sql.ErrNoRows { + util.GetLogger(req.Context()).WithError(err).Error("failed to select multiroom data for room") + return jsonerror.InternalServerError() + } + } + return util.JSONResponse{ + Code: http.StatusOK, + JSON: getLocationSyncResponse{RecentLocations: mr}, + } +} diff --git a/syncapi/routing/routing.go b/syncapi/routing/routing.go index b1283247b5..73a5e58039 100644 --- a/syncapi/routing/routing.go +++ b/syncapi/routing/routing.go @@ -205,4 +205,14 @@ func Setup( return GetMemberships(req, device, vars["roomID"], syncDB, rsAPI, true, &membership, nil, at) }), ).Methods(http.MethodGet, http.MethodOptions) + + v3mux.Handle("/rooms/{roomID}/location_sync", + httputil.MakeAuthAPI("location_sync", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } + return GetLocationSync(req, device, vars["roomID"], syncDB, rsAPI) + }), + ).Methods(http.MethodGet, http.MethodOptions) } diff --git a/syncapi/storage/postgres/multiroomcast_table.go b/syncapi/storage/postgres/multiroomcast_table.go index a7672f939a..de07616edf 100644 --- a/syncapi/storage/postgres/multiroomcast_table.go +++ b/syncapi/storage/postgres/multiroomcast_table.go @@ -25,8 +25,12 @@ WHERE v.room_id = ANY($1) AND id > $2 AND id <= $3` +const selectMaxMultiCastIDSQL = "" + + "SELECT MAX(id) FROM syncapi_multiroom_data" + type multiRoomStatements struct { - selectMultiRoomCast *sql.Stmt + selectMultiRoomCast *sql.Stmt + selectMaxMultiCastID *sql.Stmt } func NewPostgresMultiRoomCastTable(db *sql.DB) (tables.MultiRoom, error) { @@ -37,6 +41,7 @@ func NewPostgresMultiRoomCastTable(db *sql.DB) (tables.MultiRoom, error) { } return r, sqlutil.StatementList{ {&r.selectMultiRoomCast, selectMultiRoomCastSQL}, + {&r.selectMaxMultiCastID, selectMaxMultiCastIDSQL}, }.Prepare(db) } @@ -59,3 +64,11 @@ func (s *multiRoomStatements) SelectMultiRoomData(ctx context.Context, r *types. } return data, rows.Err() } + +func (s *multiRoomStatements) SelectMaxMultiRoomDataEventId( + ctx context.Context, txn *sql.Tx, +) (id int64, err error) { + stmt := sqlutil.TxStmt(txn, s.selectMaxMultiCastID) + err = stmt.QueryRowContext(ctx).Scan(&id) + return +} diff --git a/syncapi/storage/shared/storage_sync.go b/syncapi/storage/shared/storage_sync.go index aa0eb06764..5b4d528a71 100644 --- a/syncapi/storage/shared/storage_sync.go +++ b/syncapi/storage/shared/storage_sync.go @@ -824,3 +824,8 @@ func (d *DatabaseTransaction) SelectMultiRoomData(ctx context.Context, r *types. return mr, nil } + +func (d *DatabaseTransaction) SelectMaxMultiRoomDataEventId(ctx context.Context) (types.StreamPosition, error) { + id, err := d.MultiRoom.SelectMaxMultiRoomDataEventId(ctx, d.txn) + return types.StreamPosition(id), err +} diff --git a/syncapi/storage/tables/interface.go b/syncapi/storage/tables/interface.go index 6065d39c74..462380fc50 100644 --- a/syncapi/storage/tables/interface.go +++ b/syncapi/storage/tables/interface.go @@ -245,4 +245,5 @@ type Relations interface { type MultiRoom interface { SelectMultiRoomData(ctx context.Context, r *types.Range, joinedRooms []string, txn *sql.Tx) ([]*types.MultiRoomDataRow, error) + SelectMaxMultiRoomDataEventId(ctx context.Context, txn *sql.Tx) (id int64, err error) } From 656f8fe08ad18d297b370c2f08cd1f7546a81d6d Mon Sep 17 00:00:00 2001 From: Daniel Aloni Date: Sun, 9 Jul 2023 16:26:05 +0300 Subject: [PATCH 2/3] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20A=20more=20simplified?= =?UTF-8?q?=20way=20to=20query=20all=20the=20multiroom=20data=20for=20the?= =?UTF-8?q?=20room.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- syncapi/routing/location_sync.go | 10 +---- syncapi/storage/interface.go | 1 + .../storage/postgres/multiroomcast_table.go | 37 +++++++++++++------ syncapi/storage/shared/storage_sync.go | 20 ++++++++-- syncapi/storage/tables/interface.go | 2 +- 5 files changed, 47 insertions(+), 23 deletions(-) diff --git a/syncapi/routing/location_sync.go b/syncapi/routing/location_sync.go index 340726525b..68794ca244 100644 --- a/syncapi/routing/location_sync.go +++ b/syncapi/routing/location_sync.go @@ -15,7 +15,6 @@ package routing import ( - "context" "database/sql" "net/http" @@ -45,15 +44,10 @@ func GetLocationSync( logrus.WithError(err).Error("Failed to get snapshot for locations sync") return jsonerror.InternalServerError() } - id, err := snapshot.SelectMaxMultiRoomDataEventId(context.Background()) - if err != nil { - util.GetLogger(req.Context()).WithError(err).Error("failed to get max multiroom data event id") - return jsonerror.InternalServerError() - } - mr, err := snapshot.SelectMultiRoomData(req.Context(), &types.Range{From: 0, To: id}, []string{roomID}) + mr, err := snapshot.SelectAllMultiRoomDataInRoom(req.Context(), roomID) if err != nil { if err != sql.ErrNoRows { - util.GetLogger(req.Context()).WithError(err).Error("failed to select multiroom data for room") + util.GetLogger(req.Context()).WithError(err).Error("failed to select all most recent multiroom data for room") return jsonerror.InternalServerError() } } diff --git a/syncapi/storage/interface.go b/syncapi/storage/interface.go index a53822f7eb..faadbdfe33 100644 --- a/syncapi/storage/interface.go +++ b/syncapi/storage/interface.go @@ -109,6 +109,7 @@ type DatabaseTransaction interface { PresenceAfter(ctx context.Context, after types.StreamPosition, filter synctypes.EventFilter) (map[string]*types.PresenceInternal, error) RelationsFor(ctx context.Context, roomID, eventID, relType, eventType string, from, to types.StreamPosition, backwards bool, limit int) (events []types.StreamEvent, prevBatch, nextBatch string, err error) SelectMultiRoomData(ctx context.Context, r *types.Range, joinedRooms []string) (types.MultiRoom, error) + SelectAllMultiRoomDataInRoom(ctx context.Context, roomId string) (types.MultiRoom, error) } type Database interface { diff --git a/syncapi/storage/postgres/multiroomcast_table.go b/syncapi/storage/postgres/multiroomcast_table.go index de07616edf..60e4acf855 100644 --- a/syncapi/storage/postgres/multiroomcast_table.go +++ b/syncapi/storage/postgres/multiroomcast_table.go @@ -25,12 +25,15 @@ WHERE v.room_id = ANY($1) AND id > $2 AND id <= $3` -const selectMaxMultiCastIDSQL = "" + - "SELECT MAX(id) FROM syncapi_multiroom_data" +const selectAllMultiRoomCastInRoomSQL = `SELECT d.user_id, d.type, d.data, d.ts FROM syncapi_multiroom_data AS d +JOIN syncapi_multiroom_visibility AS v +ON d.user_id = v.user_id +AND concat(d.type, '.visibility') = v.type +WHERE v.room_id = $1` type multiRoomStatements struct { - selectMultiRoomCast *sql.Stmt - selectMaxMultiCastID *sql.Stmt + selectMultiRoomCast *sql.Stmt + selectAllMultiRoomCastInRoom *sql.Stmt } func NewPostgresMultiRoomCastTable(db *sql.DB) (tables.MultiRoom, error) { @@ -41,7 +44,7 @@ func NewPostgresMultiRoomCastTable(db *sql.DB) (tables.MultiRoom, error) { } return r, sqlutil.StatementList{ {&r.selectMultiRoomCast, selectMultiRoomCastSQL}, - {&r.selectMaxMultiCastID, selectMaxMultiCastIDSQL}, + {&r.selectAllMultiRoomCastInRoom, selectAllMultiRoomCastInRoomSQL}, }.Prepare(db) } @@ -65,10 +68,22 @@ func (s *multiRoomStatements) SelectMultiRoomData(ctx context.Context, r *types. return data, rows.Err() } -func (s *multiRoomStatements) SelectMaxMultiRoomDataEventId( - ctx context.Context, txn *sql.Tx, -) (id int64, err error) { - stmt := sqlutil.TxStmt(txn, s.selectMaxMultiCastID) - err = stmt.QueryRowContext(ctx).Scan(&id) - return +func (s *multiRoomStatements) SelectAllMultiRoomDataInRoom(ctx context.Context, roomId string, txn *sql.Tx) ([]*types.MultiRoomDataRow, error) { + rows, err := sqlutil.TxStmt(txn, s.selectAllMultiRoomCastInRoom).QueryContext(ctx, roomId) + if err != nil { + return nil, err + } + data := make([]*types.MultiRoomDataRow, 0) + defer internal.CloseAndLogIfError(ctx, rows, "SelectAllMultiRoomDataInRoom: rows.close() failed") + var t time.Time + for rows.Next() { + r := types.MultiRoomDataRow{} + err = rows.Scan(&r.UserId, &r.Type, &r.Data, &t) + r.Timestamp = t.UnixMilli() + if err != nil { + return nil, fmt.Errorf("rows scan: %w", err) + } + data = append(data, &r) + } + return data, rows.Err() } diff --git a/syncapi/storage/shared/storage_sync.go b/syncapi/storage/shared/storage_sync.go index 5b4d528a71..12b3df11b6 100644 --- a/syncapi/storage/shared/storage_sync.go +++ b/syncapi/storage/shared/storage_sync.go @@ -825,7 +825,21 @@ func (d *DatabaseTransaction) SelectMultiRoomData(ctx context.Context, r *types. } -func (d *DatabaseTransaction) SelectMaxMultiRoomDataEventId(ctx context.Context) (types.StreamPosition, error) { - id, err := d.MultiRoom.SelectMaxMultiRoomDataEventId(ctx, d.txn) - return types.StreamPosition(id), err +func (d *DatabaseTransaction) SelectAllMultiRoomDataInRoom(ctx context.Context, roomId string) (types.MultiRoom, error) { + rows, err := d.MultiRoom.SelectAllMultiRoomDataInRoom(ctx, roomId, d.txn) + if err != nil { + return nil, fmt.Errorf("select all multi room data in room: %w", err) + } + mr := make(types.MultiRoom, 3) + for _, row := range rows { + if mr[row.UserId] == nil { + mr[row.UserId] = make(map[string]types.MultiRoomData) + } + mr[row.UserId][row.Type] = types.MultiRoomData{ + Content: row.Data, + OriginServerTs: row.Timestamp, + } + } + return mr, nil + } diff --git a/syncapi/storage/tables/interface.go b/syncapi/storage/tables/interface.go index 462380fc50..7a66ed7dd8 100644 --- a/syncapi/storage/tables/interface.go +++ b/syncapi/storage/tables/interface.go @@ -245,5 +245,5 @@ type Relations interface { type MultiRoom interface { SelectMultiRoomData(ctx context.Context, r *types.Range, joinedRooms []string, txn *sql.Tx) ([]*types.MultiRoomDataRow, error) - SelectMaxMultiRoomDataEventId(ctx context.Context, txn *sql.Tx) (id int64, err error) + SelectAllMultiRoomDataInRoom(ctx context.Context, roomId string, txn *sql.Tx) ([]*types.MultiRoomDataRow, error) } From 20bbac5478aae2c1718aa4c3cc5ee3a356d797d5 Mon Sep 17 00:00:00 2001 From: Daniel Aloni Date: Mon, 10 Jul 2023 17:56:18 +0300 Subject: [PATCH 3/3] =?UTF-8?q?=F0=9F=94=A5=20Get=20rid=20of=20jsonerror.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- syncapi/routing/location_sync.go | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/syncapi/routing/location_sync.go b/syncapi/routing/location_sync.go index 68794ca244..40ba3f8505 100644 --- a/syncapi/routing/location_sync.go +++ b/syncapi/routing/location_sync.go @@ -18,11 +18,11 @@ import ( "database/sql" "net/http" - "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/syncapi/storage" "github.com/matrix-org/dendrite/syncapi/types" userapi "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/gomatrixserverlib/spec" "github.com/matrix-org/util" "github.com/sirupsen/logrus" ) @@ -42,13 +42,19 @@ func GetLocationSync( snapshot, err := syncDB.NewDatabaseSnapshot(req.Context()) if err != nil { logrus.WithError(err).Error("Failed to get snapshot for locations sync") - return jsonerror.InternalServerError() + return util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: spec.InternalServerError{}, + } } mr, err := snapshot.SelectAllMultiRoomDataInRoom(req.Context(), roomID) if err != nil { if err != sql.ErrNoRows { util.GetLogger(req.Context()).WithError(err).Error("failed to select all most recent multiroom data for room") - return jsonerror.InternalServerError() + return util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: spec.InternalServerError{}, + } } } return util.JSONResponse{