diff --git a/.github/workflows/docs-pr-netlify.yaml b/.github/workflows/docs-pr-netlify.yaml index b443cd87d1fb..8b2032230881 100644 --- a/.github/workflows/docs-pr-netlify.yaml +++ b/.github/workflows/docs-pr-netlify.yaml @@ -22,7 +22,7 @@ jobs: path: book - name: 📤 Deploy to Netlify - uses: matrix-org/netlify-pr-preview@v2 + uses: matrix-org/netlify-pr-preview@v3 with: path: book owner: ${{ github.event.workflow_run.head_repository.owner.login }} diff --git a/.github/workflows/docs-pr.yaml b/.github/workflows/docs-pr.yaml index 3704bd66e2ce..9cf3d340a419 100644 --- a/.github/workflows/docs-pr.yaml +++ b/.github/workflows/docs-pr.yaml @@ -6,6 +6,7 @@ on: - docs/** - book.toml - .github/workflows/docs-pr.yaml + - scripts-dev/schema_versions.py jobs: pages: @@ -13,12 +14,22 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + with: + # Fetch all history so that the schema_versions script works. + fetch-depth: 0 - name: Setup mdbook uses: peaceiris/actions-mdbook@adeb05db28a0c0004681db83893d56c0388ea9ea # v1.2.0 with: mdbook-version: '0.4.17' + - name: Setup python + uses: actions/setup-python@v4 + with: + python-version: "3.x" + + - run: "pip install 'packaging>=20.0' 'GitPython>=3.1.20'" + - name: Build the documentation # mdbook will only create an index.html if we're including docs/README.md in SUMMARY.md. # However, we're using docs/README.md for other purposes and need to pick a new page diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index c7cb2d78e504..31b9dbe3fea6 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -51,12 +51,22 @@ jobs: - pre steps: - uses: actions/checkout@v4 + with: + # Fetch all history so that the schema_versions script works. + fetch-depth: 0 - name: Setup mdbook uses: peaceiris/actions-mdbook@adeb05db28a0c0004681db83893d56c0388ea9ea # v1.2.0 with: mdbook-version: '0.4.17' + - name: Setup python + uses: actions/setup-python@v4 + with: + python-version: "3.x" + + - run: "pip install 'packaging>=20.0' 'GitPython>=3.1.20'" + - name: Build the documentation # mdbook will only create an index.html if we're including docs/README.md in SUMMARY.md. # However, we're using docs/README.md for other purposes and need to pick a new page diff --git a/.github/workflows/fix_lint.yaml b/.github/workflows/fix_lint.yaml new file mode 100644 index 000000000000..f1e35fcd99d9 --- /dev/null +++ b/.github/workflows/fix_lint.yaml @@ -0,0 +1,52 @@ +# A helper workflow to automatically fixup any linting errors on a PR. Must be +# triggered manually. + +name: Attempt to automatically fix linting errors + +on: + workflow_dispatch: + +jobs: + fixup: + name: Fix up + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Install Rust + uses: dtolnay/rust-toolchain@master + with: + # We use nightly so that `fmt` correctly groups together imports, and + # clippy correctly fixes up the benchmarks. + toolchain: nightly-2022-12-01 + components: rustfmt + - uses: Swatinem/rust-cache@v2 + + - name: Setup Poetry + uses: matrix-org/setup-python-poetry@v1 + with: + install-project: "false" + + - name: Import order (isort) + continue-on-error: true + run: poetry run isort . + + - name: Code style (black) + continue-on-error: true + run: poetry run black . + + - name: Semantic checks (ruff) + continue-on-error: true + run: poetry run ruff --fix . + + - run: cargo clippy --all-features --fix -- -D warnings + continue-on-error: true + + - run: cargo fmt + continue-on-error: true + + - uses: stefanzweifel/git-auto-commit-action@v5 + with: + commit_message: "Attempt to fix linting" diff --git a/CHANGES.md b/CHANGES.md index 40a2e086c5c5..a52173609da2 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,4 +1,6 @@ -# Synapse 1.97.0rc1 (2023-11-21) +# Synapse 1.98.0rc1 (2023-12-05) + +Synapse 1.98.0 will be the last Synapse release in 2023; the regular release cadence will resume in January 2024. Synapse will soon be forked by Element under an AGPLv3.0 licence (with CLA, for proprietary dual licensing). You can read more about this here: @@ -12,6 +14,83 @@ but we are striving to make this as seamless as possible. ### Features +- Synapse now declares support for Matrix v1.7, v1.8, and v1.9. ([\#16707](https://github.com/matrix-org/synapse/issues/16707)) +- Add `on_user_login` [module API](https://matrix-org.github.io/synapse/latest/modules/writing_a_module.html) callback allowing to execute custom code after (on) Auth. ([\#15207](https://github.com/matrix-org/synapse/issues/15207)) +- Support [MSC4069: Inhibit profile propagation](https://github.com/matrix-org/matrix-spec-proposals/pull/4069). ([\#16636](https://github.com/matrix-org/synapse/issues/16636)) +- Restore tracking of requests and monthly active users when delegating authentication via [MSC3861](https://github.com/matrix-org/synapse/pull/16672) to an OIDC provider. ([\#16672](https://github.com/matrix-org/synapse/issues/16672)) +- Add an autojoin setting for server notices rooms, so users may be joined directly instead of receiving an invite. ([\#16699](https://github.com/matrix-org/synapse/issues/16699)) +- Follow redirects when downloading media over federation (per [MSC3860](https://github.com/matrix-org/matrix-spec-proposals/pull/3860)). ([\#16701](https://github.com/matrix-org/synapse/issues/16701)) + +### Bugfixes + +- Enable refreshable tokens on the admin registration endpoint. ([\#16642](https://github.com/matrix-org/synapse/issues/16642)) +- Consistently bypass rate limits when using the server notice admin API. ([\#16670](https://github.com/matrix-org/synapse/issues/16670)) +- Fix a bug introduced in Synapse 1.7.2 where rooms whose power levels lacked an `events` field could not be upgraded. ([\#16725](https://github.com/matrix-org/synapse/issues/16725)) +- Fix `GET /_synapse/admin/v1/federation/destinations` [admin API](https://matrix-org.github.io/synapse/latest/usage/administration/admin_api/index.html) returning null (instead of 0) for `retry_last_ts` and `retry_interval`. ([\#16729](https://github.com/matrix-org/synapse/issues/16729)) + +### Improved Documentation + +- Add schema rollback information to documentation. ([\#16661](https://github.com/matrix-org/synapse/issues/16661)) +- Fix poetry version typo in the [contributors' guide](https://matrix-org.github.io/synapse/latest/development/contributing_guide.html). ([\#16695](https://github.com/matrix-org/synapse/issues/16695)) +- Switch the example UNIX socket paths to `/run`. Add HAProxy example configuration for UNIX sockets. ([\#16700](https://github.com/matrix-org/synapse/issues/16700)) +- Add documentation for how to validate the configuration file with `synapse.config` script. ([\#16714](https://github.com/matrix-org/synapse/issues/16714)) + +### Internal Changes + +- Clean-up unused tables. ([\#16522](https://github.com/matrix-org/synapse/issues/16522)) +- Reduce a little database load while processing state auth chains. ([\#16552](https://github.com/matrix-org/synapse/issues/16552)) +- Reduce database load of pruning old `user_ips`. ([\#16667](https://github.com/matrix-org/synapse/issues/16667)) +- Reduce DB load when forget on leave setting is disabled. ([\#16668](https://github.com/matrix-org/synapse/issues/16668)) +- Ignore `encryption_enabled_by_default_for_room_type` setting when creating server notices room, since the notices will be send unencrypted anyway. ([\#16677](https://github.com/matrix-org/synapse/issues/16677)) +- Correctly read the to-device stream ID on startup using SQLite. ([\#16682](https://github.com/matrix-org/synapse/issues/16682)) +- Reoranganise test files. ([\#16684](https://github.com/matrix-org/synapse/issues/16684)) +- Remove old full schema dumps which are no longer used. ([\#16697](https://github.com/matrix-org/synapse/issues/16697)) +- Raise poetry-core upper bound to <=1.8.1. This allows contributors to import Synapse after `poetry install`ing with Poetry 1.6 and above. Contributed by Mo Balaa. ([\#16702](https://github.com/matrix-org/synapse/issues/16702)) +- Add a workflow to try and automatically fixup linting in a PR. ([\#16704](https://github.com/matrix-org/synapse/issues/16704)) + + +### Updates to locked dependencies + +* Bump cryptography from 41.0.5 to 41.0.6. ([\#16703](https://github.com/matrix-org/synapse/issues/16703)) +* Bump cryptography from 41.0.6 to 41.0.7. ([\#16721](https://github.com/matrix-org/synapse/issues/16721)) +* Bump idna from 3.4 to 3.6. ([\#16720](https://github.com/matrix-org/synapse/issues/16720)) +* Bump jsonschema from 4.19.1 to 4.20.0. ([\#16692](https://github.com/matrix-org/synapse/issues/16692)) +* Bump matrix-org/netlify-pr-preview from 2 to 3. ([\#16719](https://github.com/matrix-org/synapse/issues/16719)) +* Bump phonenumbers from 8.13.23 to 8.13.26. ([\#16722](https://github.com/matrix-org/synapse/issues/16722)) +* Bump prometheus-client from 0.18.0 to 0.19.0. ([\#16691](https://github.com/matrix-org/synapse/issues/16691)) +* Bump pyasn1 from 0.5.0 to 0.5.1. ([\#16689](https://github.com/matrix-org/synapse/issues/16689)) +* Bump pydantic from 2.4.2 to 2.5.1. ([\#16663](https://github.com/matrix-org/synapse/issues/16663)) +* Bump pyo3 (0.19.2→0.20.0), pythonize (0.19.0→0.20.0) and pyo3-log (0.8.1→0.9.0). ([\#16673](https://github.com/matrix-org/synapse/issues/16673)) +* Bump pyopenssl from 23.2.0 to 23.3.0. ([\#16662](https://github.com/matrix-org/synapse/issues/16662)) +* Bump ruff from 0.1.4 to 0.1.6. ([\#16690](https://github.com/matrix-org/synapse/issues/16690)) +* Bump sentry-sdk from 1.32.0 to 1.35.0. ([\#16666](https://github.com/matrix-org/synapse/issues/16666)) +* Bump serde from 1.0.192 to 1.0.193. ([\#16693](https://github.com/matrix-org/synapse/issues/16693)) +* Bump sphinx-autodoc2 from 0.4.2 to 0.5.0. ([\#16723](https://github.com/matrix-org/synapse/issues/16723)) +* Bump types-jsonschema from 4.19.0.4 to 4.20.0.0. ([\#16724](https://github.com/matrix-org/synapse/issues/16724)) +* Bump types-pillow from 10.1.0.0 to 10.1.0.2. ([\#16664](https://github.com/matrix-org/synapse/issues/16664)) +* Bump types-psycopg2 from 2.9.21.15 to 2.9.21.16. ([\#16665](https://github.com/matrix-org/synapse/issues/16665)) +* Bump types-setuptools from 68.2.0.0 to 68.2.0.2. ([\#16688](https://github.com/matrix-org/synapse/issues/16688)) + +# Synapse 1.97.0 (2023-11-28) + +Synapse will soon be forked by Element under an AGPLv3.0 licence (with CLA, for +proprietary dual licensing). You can read more about this here: + + - https://matrix.org/blog/2023/11/06/future-of-synapse-dendrite/ + - https://element.io/blog/element-to-adopt-agplv3/ + +The Matrix.org Foundation copy of the project will be archived. Any changes needed +by server administrators will be communicated via our usual announcements channels, +but we are striving to make this as seamless as possible. + + +No significant changes since 1.97.0rc1. + + +# Synapse 1.97.0rc1 (2023-11-21) + +### Features + - Add support for asynchronous uploads as defined by [MSC2246](https://github.com/matrix-org/matrix-spec-proposals/pull/2246). Contributed by @sumnerevans at @beeper. ([\#15503](https://github.com/matrix-org/synapse/issues/15503)) - Improve the performance of some operations in multi-worker deployments. ([\#16613](https://github.com/matrix-org/synapse/issues/16613), [\#16616](https://github.com/matrix-org/synapse/issues/16616)) diff --git a/book.toml b/book.toml index fa83d86ffc1c..ed3f6151e06a 100644 --- a/book.toml +++ b/book.toml @@ -36,4 +36,7 @@ additional-css = [ "docs/website_files/indent-section-headers.css", ] additional-js = ["docs/website_files/table-of-contents.js"] -theme = "docs/website_files/theme" \ No newline at end of file +theme = "docs/website_files/theme" + +[preprocessor.schema_versions] +command = "./scripts-dev/schema_versions.py" diff --git a/changelog.d/16552.misc b/changelog.d/16552.misc deleted file mode 100644 index 73a0376df097..000000000000 --- a/changelog.d/16552.misc +++ /dev/null @@ -1 +0,0 @@ -Reduce a little database load while processing state auth chains. diff --git a/changelog.d/16642.bugfix b/changelog.d/16642.bugfix deleted file mode 100644 index e83c8b4f9d58..000000000000 --- a/changelog.d/16642.bugfix +++ /dev/null @@ -1 +0,0 @@ -Enable refreshable tokens on the admin registration endpoint. diff --git a/changelog.d/16670.bugfix b/changelog.d/16670.bugfix deleted file mode 100644 index f1369abc0607..000000000000 --- a/changelog.d/16670.bugfix +++ /dev/null @@ -1 +0,0 @@ -Consistently bypass rate limits when using the server notice admin API. diff --git a/changelog.d/16672.feature b/changelog.d/16672.feature deleted file mode 100644 index 05ecf2220733..000000000000 --- a/changelog.d/16672.feature +++ /dev/null @@ -1 +0,0 @@ -Restore tracking of requests and monthly active users when delegating authentication to an [MSC3861](https://github.com/matrix-org/synapse/pull/16672) OIDC provider. diff --git a/changelog.d/16673.misc b/changelog.d/16673.misc deleted file mode 100644 index 8b274ede71f8..000000000000 --- a/changelog.d/16673.misc +++ /dev/null @@ -1 +0,0 @@ -Bump pyo3 (0.19.2→0.20.0), pythonize (0.19.0→0.20.0) and pyo3-log (0.8.1→0.9.0). diff --git a/changelog.d/16677.misc b/changelog.d/16677.misc deleted file mode 100644 index 20c37851c396..000000000000 --- a/changelog.d/16677.misc +++ /dev/null @@ -1 +0,0 @@ -Ignore `encryption_enabled_by_default_for_room_type` setting when creating server notices room, since the notices will be send unencrypted anyway. diff --git a/changelog.d/16682.misc b/changelog.d/16682.misc deleted file mode 100644 index 071715e83a80..000000000000 --- a/changelog.d/16682.misc +++ /dev/null @@ -1 +0,0 @@ -Correctly read the to-device stream ID on startup using SQLite. diff --git a/changelog.d/16684.misc b/changelog.d/16684.misc deleted file mode 100644 index 6fb55c08a57b..000000000000 --- a/changelog.d/16684.misc +++ /dev/null @@ -1 +0,0 @@ -Reoranganise test files. diff --git a/changelog.d/16695.doc b/changelog.d/16695.doc deleted file mode 100644 index 6cb284c50114..000000000000 --- a/changelog.d/16695.doc +++ /dev/null @@ -1 +0,0 @@ -Fix poetry version typo in [contributors' guide](https://matrix-org.github.io/synapse/latest/development/contributing_guide.html). diff --git a/changelog.d/16697.misc b/changelog.d/16697.misc deleted file mode 100644 index 02fd236ab43e..000000000000 --- a/changelog.d/16697.misc +++ /dev/null @@ -1 +0,0 @@ -Remove old full schema dumps which are no longer used. diff --git a/debian/changelog b/debian/changelog index 1a4be59be1ce..b4626f741e04 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,15 @@ +matrix-synapse-py3 (1.98.0~rc1) stable; urgency=medium + + * New Synapse release 1.98.0rc1. + + -- Synapse Packaging team Tue, 05 Dec 2023 13:08:42 +0000 + +matrix-synapse-py3 (1.97.0) stable; urgency=medium + + * New Synapse release 1.97.0. + + -- Synapse Packaging team Tue, 28 Nov 2023 14:08:58 +0000 + matrix-synapse-py3 (1.97.0~rc1) stable; urgency=medium * New Synapse release 1.97.0rc1. diff --git a/docs/modules/account_validity_callbacks.md b/docs/modules/account_validity_callbacks.md index 3cd0e7219894..f5eefcd7d639 100644 --- a/docs/modules/account_validity_callbacks.md +++ b/docs/modules/account_validity_callbacks.md @@ -42,3 +42,16 @@ operations to keep track of them. (e.g. add them to a database table). The user represented by their Matrix user ID. If multiple modules implement this callback, Synapse runs them all in order. + +### `on_user_login` + +_First introduced in Synapse v1.98.0_ + +```python +async def on_user_login(user_id: str, auth_provider_type: str, auth_provider_id: str) -> None +``` + +Called after successfully login or registration of a user for cases when module needs to perform extra operations after auth. +represented by their Matrix user ID. + +If multiple modules implement this callback, Synapse runs them all in order. diff --git a/docs/reverse_proxy.md b/docs/reverse_proxy.md index fe9519b4b624..20854035d11c 100644 --- a/docs/reverse_proxy.md +++ b/docs/reverse_proxy.md @@ -181,7 +181,11 @@ frontend matrix-federation backend matrix server matrix 127.0.0.1:8008 ``` - +Example configuration, if using a UNIX socket. The configuration lines regarding the frontends do not need to be modified. +``` +backend matrix + server matrix unix@/run/synapse/main_public.sock +``` [Delegation](delegate.md) example: ``` diff --git a/docs/server_notices.md b/docs/server_notices.md index 0585665e2ecb..ac0febe8a207 100644 --- a/docs/server_notices.md +++ b/docs/server_notices.md @@ -48,6 +48,7 @@ server_notices: room_name: "Server Notices" room_avatar_url: "mxc://server.com/oumMVlgDnLYFaPVkExemNVVZ" room_topic: "Room used by your server admin to notice you of important information" + auto_join: true ``` The only compulsory setting is `system_mxid_localpart`, which defines the user @@ -57,6 +58,8 @@ room which will be created, `room_avatar_url` its avatar and `room_topic` its to `system_mxid_display_name` and `system_mxid_avatar_url` can be used to set the displayname and avatar of the Server Notices user. +`auto_join` will autojoin users to the notices room instead of sending an invite. + ## Sending notices To send server notices to users you can use the diff --git a/docs/upgrade.md b/docs/upgrade.md index ba2f7703bc75..329c9c778713 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -88,6 +88,15 @@ process, for example: dpkg -i matrix-synapse-py3_1.3.0+stretch1_amd64.deb ``` +Generally Synapse database schemas are compatible across multiple versions, once +a version of Synapse is deployed you may not be able to rollback automatically. +The following table gives the version ranges and the earliest version they can +be rolled back to. E.g. Synapse versions v1.58.0 through v1.61.1 can be rolled +back safely to v1.57.0, but starting with v1.62.0 it is only safe to rollback to +v1.61.0. + + + # Upgrading to v1.93.0 ## Minimum supported Rust version diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index 1591b42054ec..d7c852b90d99 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -33,6 +33,23 @@ In addition, configuration options referring to size use the following suffixes: For example, setting `max_avatar_size: 10M` means that Synapse will not accept files larger than 10,485,760 bytes for a user avatar. +## Config Validation + +The configuration file can be validated with the following command: +```bash +python -m synapse.config read -c +``` + +To validate the entire file, omit `read `: +```bash +python -m synapse.config -c +``` + +To see how to set other options, check the help reference: +```bash +python -m synapse.config --help +``` + ### YAML The configuration file is a [YAML](https://yaml.org/) file, which means that certain syntax rules apply if you want your config file to be read properly. A few helpful things to know: @@ -566,7 +583,7 @@ listeners: # Note that x_forwarded will default to true, when using a UNIX socket. Please see # https://matrix-org.github.io/synapse/latest/reverse_proxy.html. # - - path: /var/run/synapse/main_public.sock + - path: /run/synapse/main_public.sock type: http resources: - names: [client, federation] @@ -3817,6 +3834,8 @@ Sub-options for this setting include: * `room_name`: set the room name of the server notices room * `room_avatar_url`: set the room avatar of the server notices room _Added in Synapse 1.98.0._ * `room_topic`: set the room topic of the server notices room _Added in Synapse 1.98.0._ +* `auto_join`: boolean. If true, the user will be automatically joined to the room instead of being invited. + Defaults to false. _Added in Synapse 1.98.0._ Note that it will only be updated when a new notice event is sent. @@ -3829,6 +3848,7 @@ server_notices: room_name: "Server Notices" room_avatar_url: "mxc://server.com/oumMVlgDnLYFaPVkExemNVVZ" room_topic: "Room used by your server admin to notice you of important information" + auto_join: true ``` --- ### `enable_room_list_search` @@ -4221,9 +4241,9 @@ Example configuration(#2, for UNIX sockets): ```yaml instance_map: main: - path: /var/run/synapse/main_replication.sock + path: /run/synapse/main_replication.sock worker1: - path: /var/run/synapse/worker1_replication.sock + path: /run/synapse/worker1_replication.sock ``` --- ### `stream_writers` @@ -4409,13 +4429,13 @@ Example configuration(#2, using UNIX sockets with a `replication` listener): ```yaml worker_listeners: - type: http - path: /var/run/synapse/worker_public.sock + path: /run/synapse/worker_replication.sock resources: - - names: [client, federation] + - names: [replication] - type: http - path: /var/run/synapse/worker_replication.sock + path: /run/synapse/worker_public.sock resources: - - names: [replication] + - names: [client, federation] ``` --- ### `worker_manhole` diff --git a/poetry.lock b/poetry.lock index c10167f25705..89387db20aa0 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. [[package]] name = "alabaster" @@ -454,34 +454,34 @@ files = [ [[package]] name = "cryptography" -version = "41.0.5" +version = "41.0.7" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." optional = false python-versions = ">=3.7" files = [ - {file = "cryptography-41.0.5-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:da6a0ff8f1016ccc7477e6339e1d50ce5f59b88905585f77193ebd5068f1e797"}, - {file = "cryptography-41.0.5-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:b948e09fe5fb18517d99994184854ebd50b57248736fd4c720ad540560174ec5"}, - {file = "cryptography-41.0.5-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d38e6031e113b7421db1de0c1b1f7739564a88f1684c6b89234fbf6c11b75147"}, - {file = "cryptography-41.0.5-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e270c04f4d9b5671ebcc792b3ba5d4488bf7c42c3c241a3748e2599776f29696"}, - {file = "cryptography-41.0.5-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:ec3b055ff8f1dce8e6ef28f626e0972981475173d7973d63f271b29c8a2897da"}, - {file = "cryptography-41.0.5-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:7d208c21e47940369accfc9e85f0de7693d9a5d843c2509b3846b2db170dfd20"}, - {file = "cryptography-41.0.5-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:8254962e6ba1f4d2090c44daf50a547cd5f0bf446dc658a8e5f8156cae0d8548"}, - {file = "cryptography-41.0.5-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:a48e74dad1fb349f3dc1d449ed88e0017d792997a7ad2ec9587ed17405667e6d"}, - {file = "cryptography-41.0.5-cp37-abi3-win32.whl", hash = "sha256:d3977f0e276f6f5bf245c403156673db103283266601405376f075c849a0b936"}, - {file = "cryptography-41.0.5-cp37-abi3-win_amd64.whl", hash = "sha256:73801ac9736741f220e20435f84ecec75ed70eda90f781a148f1bad546963d81"}, - {file = "cryptography-41.0.5-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:3be3ca726e1572517d2bef99a818378bbcf7d7799d5372a46c79c29eb8d166c1"}, - {file = "cryptography-41.0.5-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:e886098619d3815e0ad5790c973afeee2c0e6e04b4da90b88e6bd06e2a0b1b72"}, - {file = "cryptography-41.0.5-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:573eb7128cbca75f9157dcde974781209463ce56b5804983e11a1c462f0f4e88"}, - {file = "cryptography-41.0.5-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:0c327cac00f082013c7c9fb6c46b7cc9fa3c288ca702c74773968173bda421bf"}, - {file = "cryptography-41.0.5-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:227ec057cd32a41c6651701abc0328135e472ed450f47c2766f23267b792a88e"}, - {file = "cryptography-41.0.5-pp38-pypy38_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:22892cc830d8b2c89ea60148227631bb96a7da0c1b722f2aac8824b1b7c0b6b8"}, - {file = "cryptography-41.0.5-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:5a70187954ba7292c7876734183e810b728b4f3965fbe571421cb2434d279179"}, - {file = "cryptography-41.0.5-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:88417bff20162f635f24f849ab182b092697922088b477a7abd6664ddd82291d"}, - {file = "cryptography-41.0.5-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:c707f7afd813478e2019ae32a7c49cd932dd60ab2d2a93e796f68236b7e1fbf1"}, - {file = "cryptography-41.0.5-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:580afc7b7216deeb87a098ef0674d6ee34ab55993140838b14c9b83312b37b86"}, - {file = "cryptography-41.0.5-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:fba1e91467c65fe64a82c689dc6cf58151158993b13eb7a7f3f4b7f395636723"}, - {file = "cryptography-41.0.5-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:0d2a6a598847c46e3e321a7aef8af1436f11c27f1254933746304ff014664d84"}, - {file = "cryptography-41.0.5.tar.gz", hash = "sha256:392cb88b597247177172e02da6b7a63deeff1937fa6fec3bbf902ebd75d97ec7"}, + {file = "cryptography-41.0.7-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:3c78451b78313fa81607fa1b3f1ae0a5ddd8014c38a02d9db0616133987b9cdf"}, + {file = "cryptography-41.0.7-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:928258ba5d6f8ae644e764d0f996d61a8777559f72dfeb2eea7e2fe0ad6e782d"}, + {file = "cryptography-41.0.7-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5a1b41bc97f1ad230a41657d9155113c7521953869ae57ac39ac7f1bb471469a"}, + {file = "cryptography-41.0.7-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:841df4caa01008bad253bce2a6f7b47f86dc9f08df4b433c404def869f590a15"}, + {file = "cryptography-41.0.7-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:5429ec739a29df2e29e15d082f1d9ad683701f0ec7709ca479b3ff2708dae65a"}, + {file = "cryptography-41.0.7-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:43f2552a2378b44869fe8827aa19e69512e3245a219104438692385b0ee119d1"}, + {file = "cryptography-41.0.7-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:af03b32695b24d85a75d40e1ba39ffe7db7ffcb099fe507b39fd41a565f1b157"}, + {file = "cryptography-41.0.7-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:49f0805fc0b2ac8d4882dd52f4a3b935b210935d500b6b805f321addc8177406"}, + {file = "cryptography-41.0.7-cp37-abi3-win32.whl", hash = "sha256:f983596065a18a2183e7f79ab3fd4c475205b839e02cbc0efbbf9666c4b3083d"}, + {file = "cryptography-41.0.7-cp37-abi3-win_amd64.whl", hash = "sha256:90452ba79b8788fa380dfb587cca692976ef4e757b194b093d845e8d99f612f2"}, + {file = "cryptography-41.0.7-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:079b85658ea2f59c4f43b70f8119a52414cdb7be34da5d019a77bf96d473b960"}, + {file = "cryptography-41.0.7-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:b640981bf64a3e978a56167594a0e97db71c89a479da8e175d8bb5be5178c003"}, + {file = "cryptography-41.0.7-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:e3114da6d7f95d2dee7d3f4eec16dacff819740bbab931aff8648cb13c5ff5e7"}, + {file = "cryptography-41.0.7-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:d5ec85080cce7b0513cfd233914eb8b7bbd0633f1d1703aa28d1dd5a72f678ec"}, + {file = "cryptography-41.0.7-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:7a698cb1dac82c35fcf8fe3417a3aaba97de16a01ac914b89a0889d364d2f6be"}, + {file = "cryptography-41.0.7-pp38-pypy38_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:37a138589b12069efb424220bf78eac59ca68b95696fc622b6ccc1c0a197204a"}, + {file = "cryptography-41.0.7-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:68a2dec79deebc5d26d617bfdf6e8aab065a4f34934b22d3b5010df3ba36612c"}, + {file = "cryptography-41.0.7-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:09616eeaef406f99046553b8a40fbf8b1e70795a91885ba4c96a70793de5504a"}, + {file = "cryptography-41.0.7-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:48a0476626da912a44cc078f9893f292f0b3e4c739caf289268168d8f4702a39"}, + {file = "cryptography-41.0.7-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:c7f3201ec47d5207841402594f1d7950879ef890c0c495052fa62f58283fde1a"}, + {file = "cryptography-41.0.7-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:c5ca78485a255e03c32b513f8c2bc39fedb7f5c5f8535545bdc223a03b24f248"}, + {file = "cryptography-41.0.7-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:d6c391c021ab1f7a82da5d8d0b3cee2f4b2c455ec86c8aebbc84837a631ff309"}, + {file = "cryptography-41.0.7.tar.gz", hash = "sha256:13f93ce9bea8016c253b34afc6bd6a75993e5c40672ed5405a9c832f0d4a00bc"}, ] [package.dependencies] @@ -712,13 +712,13 @@ idna = ">=2.5" [[package]] name = "idna" -version = "3.4" +version = "3.6" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.5" files = [ - {file = "idna-3.4-py3-none-any.whl", hash = "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"}, - {file = "idna-3.4.tar.gz", hash = "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4"}, + {file = "idna-3.6-py3-none-any.whl", hash = "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f"}, + {file = "idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca"}, ] [[package]] @@ -1611,13 +1611,13 @@ files = [ [[package]] name = "phonenumbers" -version = "8.13.23" +version = "8.13.26" description = "Python version of Google's common library for parsing, formatting, storing and validating international phone numbers." optional = false python-versions = "*" files = [ - {file = "phonenumbers-8.13.23-py2.py3-none-any.whl", hash = "sha256:34d6cb279dd4a64714e324c71350f96e5bda3237be28d11b4c555c44701544cd"}, - {file = "phonenumbers-8.13.23.tar.gz", hash = "sha256:869e44fcaaf276eca6b953a401e2b27d57461f3a18a66cf5f13377e7bb0e228c"}, + {file = "phonenumbers-8.13.26-py2.py3-none-any.whl", hash = "sha256:b2308c9c5750b8f10dd30d94547afd66bce60ac5e93aff227f95740557f32752"}, + {file = "phonenumbers-8.13.26.tar.gz", hash = "sha256:937d70aeceb317f5831dfec28de855a60260ef4a9d551964bec8e7a7d0cf81cd"}, ] [[package]] @@ -2691,17 +2691,17 @@ test = ["cython", "filelock", "html5lib", "pytest (>=4.6)"] [[package]] name = "sphinx-autodoc2" -version = "0.4.2" +version = "0.5.0" description = "Analyse a python project and create documentation for it." optional = false python-versions = ">=3.8" files = [ - {file = "sphinx-autodoc2-0.4.2.tar.gz", hash = "sha256:06da226a25a4339e173b34bb0e590e0ba9b4570b414796140aee1939d09acb3a"}, - {file = "sphinx_autodoc2-0.4.2-py3-none-any.whl", hash = "sha256:00835ba8c980b9c510ea794c3e2060e5a254a74c6c22badc9bfd3642dc1034b4"}, + {file = "sphinx_autodoc2-0.5.0-py3-none-any.whl", hash = "sha256:e867013b1512f9d6d7e6f6799f8b537d6884462acd118ef361f3f619a60b5c9e"}, + {file = "sphinx_autodoc2-0.5.0.tar.gz", hash = "sha256:7d76044aa81d6af74447080182b6868c7eb066874edc835e8ddf810735b6565a"}, ] [package.dependencies] -astroid = ">=2.7" +astroid = ">=2.7,<4" tomli = {version = "*", markers = "python_version < \"3.11\""} typing-extensions = "*" @@ -2709,7 +2709,7 @@ typing-extensions = "*" cli = ["typer[all]"] docs = ["furo", "myst-parser", "sphinx (>=4.0.0)"] sphinx = ["sphinx (>=4.0.0)"] -testing = ["pytest", "pytest-cov", "pytest-regressions", "sphinx (>=4.0.0)"] +testing = ["pytest", "pytest-cov", "pytest-regressions", "sphinx (>=4.0.0,<7)"] [[package]] name = "sphinx-basic-ng" @@ -3054,13 +3054,13 @@ files = [ [[package]] name = "types-jsonschema" -version = "4.19.0.4" +version = "4.20.0.0" description = "Typing stubs for jsonschema" optional = false python-versions = ">=3.8" files = [ - {file = "types-jsonschema-4.19.0.4.tar.gz", hash = "sha256:994feb6632818259c4b5dbd733867824cb475029a6abc2c2b5201a2268b6e7d2"}, - {file = "types_jsonschema-4.19.0.4-py3-none-any.whl", hash = "sha256:b73c3f4ba3cd8108602d1198a438e2698d5eb6b9db206ed89a33e24729b0abe7"}, + {file = "types-jsonschema-4.20.0.0.tar.gz", hash = "sha256:0de1032d243f1d3dba8b745ad84efe8c1af71665a9deb1827636ac535dcb79c1"}, + {file = "types_jsonschema-4.20.0.0-py3-none-any.whl", hash = "sha256:e6d5df18aaca4412f0aae246a294761a92040e93d7bc840f002b7329a8b72d26"}, ] [package.dependencies] @@ -3432,4 +3432,4 @@ user-search = ["pyicu"] [metadata] lock-version = "2.0" python-versions = "^3.8.0" -content-hash = "2924e80a14b32b430e70bafbd2b00893a365d9c3836c026296f4af0b9579e604" +content-hash = "57716a9580b3493c3d2038492a6d4c36d1d16a79c5a0880b6eadcaf681503d3a" diff --git a/pyproject.toml b/pyproject.toml index df683a146f67..a3eb6111e023 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -96,7 +96,7 @@ module-name = "synapse.synapse_rust" [tool.poetry] name = "matrix-synapse" -version = "1.97.0rc1" +version = "1.98.0rc1" description = "Homeserver for the Matrix decentralised comms protocol" authors = ["Matrix.org Team and Contributors "] license = "Apache-2.0" @@ -370,7 +370,7 @@ optional = true [tool.poetry.group.dev-docs.dependencies] sphinx = {version = "^6.1", python = "^3.8"} -sphinx-autodoc2 = {version = "^0.4.2", python = "^3.8"} +sphinx-autodoc2 = {version = ">=0.4.2,<0.6.0", python = "^3.8"} myst-parser = {version = "^1.0.0", python = "^3.8"} furo = ">=2022.12.7,<2024.0.0" @@ -382,7 +382,7 @@ furo = ">=2022.12.7,<2024.0.0" # runtime errors caused by build system changes. # We are happy to raise these upper bounds upon request, # provided we check that it's safe to do so (i.e. that CI passes). -requires = ["poetry-core>=1.1.0,<=1.7.0", "setuptools_rust>=1.3,<=1.8.1"] +requires = ["poetry-core>=1.1.0,<=1.8.1", "setuptools_rust>=1.3,<=1.8.1"] build-backend = "poetry.core.masonry.api" diff --git a/rust/src/push/mod.rs b/rust/src/push/mod.rs index 5e1e8e1abbec..68d4227baa81 100644 --- a/rust/src/push/mod.rs +++ b/rust/src/push/mod.rs @@ -296,8 +296,7 @@ impl<'source> FromPyObject<'source> for JsonValue { match l.iter().map(SimpleJsonValue::extract).collect() { Ok(a) => Ok(JsonValue::Array(a)), Err(e) => Err(PyTypeError::new_err(format!( - "Can't convert to JsonValue::Array: {}", - e + "Can't convert to JsonValue::Array: {e}" ))), } } else if let Ok(v) = SimpleJsonValue::extract(ob) { diff --git a/scripts-dev/schema_versions.py b/scripts-dev/schema_versions.py new file mode 100755 index 000000000000..5fd73251cdcb --- /dev/null +++ b/scripts-dev/schema_versions.py @@ -0,0 +1,181 @@ +#!/usr/bin/env python +# Copyright 2023 The Matrix.org Foundation C.I.C. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A script to calculate which versions of Synapse have backwards-compatible +database schemas. It creates a Markdown table of Synapse versions and the earliest +compatible version. + +It is compatible with the mdbook protocol for preprocessors (see +https://rust-lang.github.io/mdBook/for_developers/preprocessors.html#implementing-a-preprocessor-with-a-different-language): + +Exit 0 to denote support for all renderers: + + ./scripts-dev/schema_versions.py supports + +Parse a JSON list from stdin and add the table to the proper documetnation page: + + ./scripts-dev/schema_versions.py + +Additionally, the script supports dumping the table to stdout for debugging: + + ./scripts-dev/schema_versions.py dump +""" + +import io +import json +import sys +from collections import defaultdict +from typing import Any, Dict, Iterator, Optional, Tuple + +import git +from packaging import version + +# The schema version has moved around over the years. +SCHEMA_VERSION_FILES = ( + "synapse/storage/schema/__init__.py", + "synapse/storage/prepare_database.py", + "synapse/storage/__init__.py", + "synapse/app/homeserver.py", +) + + +# Skip versions of Synapse < v1.0, they're old and essentially not +# compatible with today's federation. +OLDEST_SHOWN_VERSION = version.parse("v1.0") + + +def get_schema_versions(tag: git.Tag) -> Tuple[Optional[int], Optional[int]]: + """Get the schema and schema compat versions for a tag.""" + schema_version = None + schema_compat_version = None + + for file in SCHEMA_VERSION_FILES: + try: + schema_file = tag.commit.tree / file + except KeyError: + continue + + # We (usually) can't execute the code since it might have unknown imports. + if file != "synapse/storage/schema/__init__.py": + with io.BytesIO(schema_file.data_stream.read()) as f: + for line in f.readlines(): + if line.startswith(b"SCHEMA_VERSION"): + schema_version = int(line.split()[2]) + + # Bail early. + break + else: + # SCHEMA_COMPAT_VERSION is sometimes across multiple lines, the easist + # thing to do is exec the code. Luckily it has only ever existed in + # a file which imports nothing else from Synapse. + locals: Dict[str, Any] = {} + exec(schema_file.data_stream.read().decode("utf-8"), {}, locals) + schema_version = locals["SCHEMA_VERSION"] + schema_compat_version = locals.get("SCHEMA_COMPAT_VERSION") + + return schema_version, schema_compat_version + + +def get_tags(repo: git.Repo) -> Iterator[git.Tag]: + """Return an iterator of tags sorted by version.""" + tags = [] + for tag in repo.tags: + # All "real" Synapse tags are of the form vX.Y.Z. + if not tag.name.startswith("v"): + continue + + # There's a weird tag from the initial react UI. + if tag.name == "v0.1": + continue + + try: + tag_version = version.parse(tag.name) + except version.InvalidVersion: + # Skip invalid versions. + continue + + # Skip pre- and post-release versions. + if tag_version.is_prerelease or tag_version.is_postrelease or tag_version.local: + continue + + # Skip old versions. + if tag_version < OLDEST_SHOWN_VERSION: + continue + + tags.append((tag_version, tag)) + + # Sort based on the version number (not lexically). + return (tag for _, tag in sorted(tags, key=lambda t: t[0])) + + +def calculate_version_chart() -> str: + repo = git.Repo(path=".") + + # Map of schema version -> Synapse versions which are at that schema version. + schema_versions = defaultdict(list) + # Map of schema version -> Synapse versions which are compatible with that + # schema version. + schema_compat_versions = defaultdict(list) + + # Find ranges of versions which are compatible with a schema version. + # + # There are two modes of operation: + # + # 1. Pre-schema_compat_version (i.e. schema_compat_version of None), then + # Synapse is compatible up/downgrading to a version with + # schema_version >= its current version. + # + # 2. Post-schema_compat_version (i.e. schema_compat_version is *not* None), + # then Synapse is compatible up/downgrading to a version with + # schema version >= schema_compat_version. + # + # This is more generous and avoids versions that cannot be rolled back. + # + # See https://github.com/matrix-org/synapse/pull/9933 which was included in v1.37.0. + for tag in get_tags(repo): + schema_version, schema_compat_version = get_schema_versions(tag) + + # If a schema compat version is given, prefer that over the schema version. + schema_versions[schema_version].append(tag.name) + schema_compat_versions[schema_compat_version or schema_version].append(tag.name) + + # Generate a table which maps the latest Synapse version compatible with each + # schema version. + result = f"| {'Versions': ^19} | Compatible version |\n" + result += f"|{'-' * (19 + 2)}|{'-' * (18 + 2)}|\n" + for schema_version, synapse_versions in schema_compat_versions.items(): + result += f"| {synapse_versions[0] + ' – ' + synapse_versions[-1]: ^19} | {schema_versions[schema_version][0]: ^18} |\n" + + return result + + +if __name__ == "__main__": + if len(sys.argv) == 3 and sys.argv[1] == "supports": + # We don't care about the renderer which is being used, which is the second argument. + sys.exit(0) + elif len(sys.argv) == 2 and sys.argv[1] == "dump": + print(calculate_version_chart()) + else: + # Expect JSON data on stdin. + context, book = json.load(sys.stdin) + + for section in book["sections"]: + if "Chapter" in section and section["Chapter"]["path"] == "upgrade.md": + section["Chapter"]["content"] = section["Chapter"]["content"].replace( + "", calculate_version_chart() + ) + + # Print the result back out to stdout. + print(json.dumps(book)) diff --git a/synapse/config/experimental.py b/synapse/config/experimental.py index 9f830e709410..6b9febe5a737 100644 --- a/synapse/config/experimental.py +++ b/synapse/config/experimental.py @@ -419,3 +419,7 @@ def read_config(self, config: JsonDict, **kwargs: Any) -> None: self.msc4028_push_encrypted_events = experimental.get( "msc4028_push_encrypted_events", False ) + + self.msc4069_profile_inhibit_propagation = experimental.get( + "msc4069_profile_inhibit_propagation", False + ) diff --git a/synapse/config/server_notices.py b/synapse/config/server_notices.py index c91563cfb953..79f365cad525 100644 --- a/synapse/config/server_notices.py +++ b/synapse/config/server_notices.py @@ -58,6 +58,7 @@ def __init__(self, *args: Any): self.server_notices_room_name: Optional[str] = None self.server_notices_room_avatar_url: Optional[str] = None self.server_notices_room_topic: Optional[str] = None + self.server_notices_auto_join: bool = False def read_config(self, config: JsonDict, **kwargs: Any) -> None: c = config.get("server_notices") @@ -74,3 +75,4 @@ def read_config(self, config: JsonDict, **kwargs: Any) -> None: self.server_notices_room_name = c.get("room_name", "Server Notices") self.server_notices_room_avatar_url = c.get("room_avatar_url", None) self.server_notices_room_topic = c.get("room_topic", None) + self.server_notices_auto_join = c.get("auto_join", False) diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index 1a7fa175ec63..0ba03b0d0540 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -21,6 +21,7 @@ TYPE_CHECKING, AbstractSet, Awaitable, + BinaryIO, Callable, Collection, Container, @@ -1862,6 +1863,43 @@ def filter_user_id(user_id: str) -> bool: return filtered_statuses, filtered_failures + async def download_media( + self, + destination: str, + media_id: str, + output_stream: BinaryIO, + max_size: int, + max_timeout_ms: int, + ) -> Tuple[int, Dict[bytes, List[bytes]]]: + try: + return await self.transport_layer.download_media_v3( + destination, + media_id, + output_stream=output_stream, + max_size=max_size, + max_timeout_ms=max_timeout_ms, + ) + except HttpResponseException as e: + # If an error is received that is due to an unrecognised endpoint, + # fallback to the r0 endpoint. Otherwise, consider it a legitimate error + # and raise. + if not is_unknown_endpoint(e): + raise + + logger.debug( + "Couldn't download media %s/%s with the v3 API, falling back to the r0 API", + destination, + media_id, + ) + + return await self.transport_layer.download_media_r0( + destination, + media_id, + output_stream=output_stream, + max_size=max_size, + max_timeout_ms=max_timeout_ms, + ) + @attr.s(frozen=True, slots=True, auto_attribs=True) class TimestampToEventResponse: diff --git a/synapse/federation/transport/client.py b/synapse/federation/transport/client.py index fab480071716..5e36638b0a65 100644 --- a/synapse/federation/transport/client.py +++ b/synapse/federation/transport/client.py @@ -18,6 +18,7 @@ from typing import ( TYPE_CHECKING, Any, + BinaryIO, Callable, Collection, Dict, @@ -804,6 +805,58 @@ async def get_account_status( destination=destination, path=path, data={"user_ids": user_ids} ) + async def download_media_r0( + self, + destination: str, + media_id: str, + output_stream: BinaryIO, + max_size: int, + max_timeout_ms: int, + ) -> Tuple[int, Dict[bytes, List[bytes]]]: + path = f"/_matrix/media/r0/download/{destination}/{media_id}" + + return await self.client.get_file( + destination, + path, + output_stream=output_stream, + max_size=max_size, + args={ + # tell the remote server to 404 if it doesn't + # recognise the server_name, to make sure we don't + # end up with a routing loop. + "allow_remote": "false", + "timeout_ms": str(max_timeout_ms), + }, + ) + + async def download_media_v3( + self, + destination: str, + media_id: str, + output_stream: BinaryIO, + max_size: int, + max_timeout_ms: int, + ) -> Tuple[int, Dict[bytes, List[bytes]]]: + path = f"/_matrix/media/v3/download/{destination}/{media_id}" + + return await self.client.get_file( + destination, + path, + output_stream=output_stream, + max_size=max_size, + args={ + # tell the remote server to 404 if it doesn't + # recognise the server_name, to make sure we don't + # end up with a routing loop. + "allow_remote": "false", + "timeout_ms": str(max_timeout_ms), + # Matrix 1.7 allows for this to redirect to another URL, this should + # just be ignored for an old homeserver, so always provide it. + "allow_redirect": "true", + }, + follow_redirects=True, + ) + def _create_path(federation_prefix: str, path: str, *args: str) -> str: """ diff --git a/synapse/handlers/account_validity.py b/synapse/handlers/account_validity.py index 6c2a49a3b91f..c66bb6364f08 100644 --- a/synapse/handlers/account_validity.py +++ b/synapse/handlers/account_validity.py @@ -98,6 +98,22 @@ async def on_user_registration(self, user_id: str) -> None: for callback in self._module_api_callbacks.on_user_registration_callbacks: await callback(user_id) + async def on_user_login( + self, + user_id: str, + auth_provider_type: Optional[str], + auth_provider_id: Optional[str], + ) -> None: + """Tell third-party modules about a user logins. + + Args: + user_id: The mxID of the user. + auth_provider_type: The type of login. + auth_provider_id: The ID of the auth provider. + """ + for callback in self._module_api_callbacks.on_user_login_callbacks: + await callback(user_id, auth_provider_type, auth_provider_id) + @wrap_as_background_process("send_renewals") async def _send_renewal_emails(self) -> None: """Gets the list of users whose account is expiring in the amount of time diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index 2b0c50513095..89cbaff864d4 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -212,6 +212,7 @@ def __init__(self, hs: "HomeServer"): self._password_enabled_for_reauth = hs.config.auth.password_enabled_for_reauth self._password_localdb_enabled = hs.config.auth.password_localdb_enabled self._third_party_rules = hs.get_module_api_callbacks().third_party_event_rules + self._account_validity_handler = hs.get_account_validity_handler() # Ratelimiter for failed auth during UIA. Uses same ratelimit config # as per `rc_login.failed_attempts`. @@ -1783,6 +1784,13 @@ async def complete_sso_login( client_redirect_url, "loginToken", login_token ) + # Run post-login module callback handlers + await self._account_validity_handler.on_user_login( + user_id=registered_user_id, + auth_provider_type=LoginType.SSO, + auth_provider_id=auth_provider_id, + ) + # if the client is whitelisted, we can redirect straight to it if client_redirect_url.startswith(self._whitelisted_sso_clients): request.redirect(redirect_url) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 811a41f161b6..25dd96416a66 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -693,13 +693,9 @@ async def create_event( if require_consent and not is_exempt: await self.assert_accepted_privacy_policy(requester) - # Save the access token ID, the device ID and the transaction ID in the event - # internal metadata. This is useful to determine if we should echo the - # transaction_id in events. + # Save the the device ID and the transaction ID in the event internal metadata. + # This is useful to determine if we should echo the transaction_id in events. # See `synapse.events.utils.EventClientSerializer.serialize_event` - if requester.access_token_id is not None: - builder.internal_metadata.token_id = requester.access_token_id - if requester.device_id is not None: builder.internal_metadata.device_id = requester.device_id diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py index 1027fbfd2839..e043fd53228b 100644 --- a/synapse/handlers/profile.py +++ b/synapse/handlers/profile.py @@ -129,6 +129,7 @@ async def set_displayname( new_displayname: str, by_admin: bool = False, deactivation: bool = False, + propagate: bool = True, ) -> None: """Set the displayname of a user @@ -138,6 +139,7 @@ async def set_displayname( new_displayname: The displayname to give this user. by_admin: Whether this change was made by an administrator. deactivation: Whether this change was made while deactivating the user. + propagate: Whether this change also applies to the user's membership events. """ if not self.hs.is_mine(target_user): raise SynapseError(400, "User is not hosted on this homeserver") @@ -188,7 +190,8 @@ async def set_displayname( target_user.to_string(), profile, by_admin, deactivation ) - await self._update_join_states(requester, target_user) + if propagate: + await self._update_join_states(requester, target_user) async def get_avatar_url(self, target_user: UserID) -> Optional[str]: if self.hs.is_mine(target_user): @@ -221,6 +224,7 @@ async def set_avatar_url( new_avatar_url: str, by_admin: bool = False, deactivation: bool = False, + propagate: bool = True, ) -> None: """Set a new avatar URL for a user. @@ -230,6 +234,7 @@ async def set_avatar_url( new_avatar_url: The avatar URL to give this user. by_admin: Whether this change was made by an administrator. deactivation: Whether this change was made while deactivating the user. + propagate: Whether this change also applies to the user's membership events. """ if not self.hs.is_mine(target_user): raise SynapseError(400, "User is not hosted on this homeserver") @@ -278,7 +283,8 @@ async def set_avatar_url( target_user.to_string(), profile, by_admin, deactivation ) - await self._update_join_states(requester, target_user) + if propagate: + await self._update_join_states(requester, target_user) @cached() async def check_avatar_size_and_mime_type(self, mxc: str) -> bool: diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index f865bed1ec2c..2823ca6f0d05 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -549,7 +549,7 @@ async def clone_existing_room( except (TypeError, ValueError): ban = 50 needed_power_level = max( - state_default_int, ban, max(event_power_levels.values()) + state_default_int, ban, max(event_power_levels.values(), default=0) ) # Get the user's current power level, this matches the logic in get_user_power_level, diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py index eddc2af9ba4c..00c2dd854d9f 100644 --- a/synapse/handlers/room_member.py +++ b/synapse/handlers/room_member.py @@ -2111,9 +2111,14 @@ async def _unsafe_process(self) -> None: self.pos = room_max_stream_ordering if not self._hs.config.room.forget_on_leave: - # Update the processing position, so that if the server admin turns the - # feature on at a later date, we don't decide to forget every room that - # has ever been left in the past. + # Update the processing position, so that if the server admin turns + # the feature on at a later date, we don't decide to forget every + # room that has ever been left in the past. + # + # We wait for a short time so that we don't "tight" loop just + # keeping the table up to date. + await self._clock.sleep(0.5) + self.pos = self._store.get_room_max_stream_ordering() await self._store.update_room_forgetter_stream_pos(self.pos) return diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index d5013e8e97c1..cc1db763ae4e 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -153,12 +153,18 @@ class MatrixFederationRequest: """Query arguments. """ - txn_id: Optional[str] = None - """Unique ID for this request (for logging) + txn_id: str = attr.ib(init=False) + """Unique ID for this request (for logging), this is autogenerated. """ - uri: bytes = attr.ib(init=False) - """The URI of this request + uri: bytes = b"" + """The URI of this request, usually generated from the above information. + """ + + _generate_uri: bool = True + """True to automatically generate the uri field based on the above information. + + Set to False if manually configuring the URI. """ def __attrs_post_init__(self) -> None: @@ -168,22 +174,23 @@ def __attrs_post_init__(self) -> None: object.__setattr__(self, "txn_id", txn_id) - destination_bytes = self.destination.encode("ascii") - path_bytes = self.path.encode("ascii") - query_bytes = encode_query_args(self.query) - - # The object is frozen so we can pre-compute this. - uri = urllib.parse.urlunparse( - ( - b"matrix-federation", - destination_bytes, - path_bytes, - None, - query_bytes, - b"", + if self._generate_uri: + destination_bytes = self.destination.encode("ascii") + path_bytes = self.path.encode("ascii") + query_bytes = encode_query_args(self.query) + + # The object is frozen so we can pre-compute this. + uri = urllib.parse.urlunparse( + ( + b"matrix-federation", + destination_bytes, + path_bytes, + None, + query_bytes, + b"", + ) ) - ) - object.__setattr__(self, "uri", uri) + object.__setattr__(self, "uri", uri) def get_json(self) -> Optional[JsonDict]: if self.json_callback: @@ -513,6 +520,7 @@ async def _send_request( ignore_backoff: bool = False, backoff_on_404: bool = False, backoff_on_all_error_codes: bool = False, + follow_redirects: bool = False, ) -> IResponse: """ Sends a request to the given server. @@ -555,6 +563,9 @@ async def _send_request( backoff_on_404: Back off if we get a 404 backoff_on_all_error_codes: Back off if we get any error response + follow_redirects: True to follow the Location header of 307/308 redirect + responses. This does not recurse. + Returns: Resolves with the HTTP response object on success. @@ -714,6 +725,26 @@ async def _send_request( response.code, response_phrase, ) + elif ( + response.code in (307, 308) + and follow_redirects + and response.headers.hasHeader("Location") + ): + # The Location header *might* be relative so resolve it. + location = response.headers.getRawHeaders(b"Location")[0] + new_uri = urllib.parse.urljoin(request.uri, location) + + return await self._send_request( + attr.evolve(request, uri=new_uri, generate_uri=False), + retry_on_dns_fail, + timeout, + long_retries, + ignore_backoff, + backoff_on_404, + backoff_on_all_error_codes, + # Do not continue following redirects. + follow_redirects=False, + ) else: logger.info( "{%s} [%s] Got response headers: %d %s", @@ -1383,6 +1414,7 @@ async def get_file( retry_on_dns_fail: bool = True, max_size: Optional[int] = None, ignore_backoff: bool = False, + follow_redirects: bool = False, ) -> Tuple[int, Dict[bytes, List[bytes]]]: """GETs a file from a given homeserver Args: @@ -1392,6 +1424,8 @@ async def get_file( args: Optional dictionary used to create the query string. ignore_backoff: true to ignore the historical backoff data and try the request anyway. + follow_redirects: True to follow the Location header of 307/308 redirect + responses. This does not recurse. Returns: Resolves with an (int,dict) tuple of @@ -1412,7 +1446,10 @@ async def get_file( ) response = await self._send_request( - request, retry_on_dns_fail=retry_on_dns_fail, ignore_backoff=ignore_backoff + request, + retry_on_dns_fail=retry_on_dns_fail, + ignore_backoff=ignore_backoff, + follow_redirects=follow_redirects, ) headers = dict(response.headers.getAllRawHeaders()) diff --git a/synapse/media/media_repository.py b/synapse/media/media_repository.py index bf976b9e7c2c..d62af22adb70 100644 --- a/synapse/media/media_repository.py +++ b/synapse/media/media_repository.py @@ -77,7 +77,7 @@ class MediaRepository: def __init__(self, hs: "HomeServer"): self.hs = hs self.auth = hs.get_auth() - self.client = hs.get_federation_http_client() + self.client = hs.get_federation_client() self.clock = hs.get_clock() self.server_name = hs.hostname self.store = hs.get_datastores().main @@ -644,22 +644,13 @@ async def _download_remote_file( file_info = FileInfo(server_name=server_name, file_id=file_id) with self.media_storage.store_into_file(file_info) as (f, fname, finish): - request_path = "/".join( - ("/_matrix/media/r0/download", server_name, media_id) - ) try: - length, headers = await self.client.get_file( + length, headers = await self.client.download_media( server_name, - request_path, + media_id, output_stream=f, max_size=self.max_upload_size, - args={ - # tell the remote server to 404 if it doesn't - # recognise the server_name, to make sure we don't - # end up with a routing loop. - "allow_remote": "false", - "timeout_ms": str(max_timeout_ms), - }, + max_timeout_ms=max_timeout_ms, ) except RequestSendFailed as e: logger.warning( diff --git a/synapse/module_api/__init__.py b/synapse/module_api/__init__.py index 812144a128a0..6ee53511f2f6 100644 --- a/synapse/module_api/__init__.py +++ b/synapse/module_api/__init__.py @@ -80,6 +80,7 @@ ON_LEGACY_ADMIN_REQUEST, ON_LEGACY_RENEW_CALLBACK, ON_LEGACY_SEND_MAIL_CALLBACK, + ON_USER_LOGIN_CALLBACK, ON_USER_REGISTRATION_CALLBACK, ) from synapse.module_api.callbacks.spamchecker_callbacks import ( @@ -334,6 +335,7 @@ def register_account_validity_callbacks( *, is_user_expired: Optional[IS_USER_EXPIRED_CALLBACK] = None, on_user_registration: Optional[ON_USER_REGISTRATION_CALLBACK] = None, + on_user_login: Optional[ON_USER_LOGIN_CALLBACK] = None, on_legacy_send_mail: Optional[ON_LEGACY_SEND_MAIL_CALLBACK] = None, on_legacy_renew: Optional[ON_LEGACY_RENEW_CALLBACK] = None, on_legacy_admin_request: Optional[ON_LEGACY_ADMIN_REQUEST] = None, @@ -345,6 +347,7 @@ def register_account_validity_callbacks( return self._callbacks.account_validity.register_callbacks( is_user_expired=is_user_expired, on_user_registration=on_user_registration, + on_user_login=on_user_login, on_legacy_send_mail=on_legacy_send_mail, on_legacy_renew=on_legacy_renew, on_legacy_admin_request=on_legacy_admin_request, diff --git a/synapse/module_api/callbacks/account_validity_callbacks.py b/synapse/module_api/callbacks/account_validity_callbacks.py index 531d0c9ddcf4..cbfdfbd3d186 100644 --- a/synapse/module_api/callbacks/account_validity_callbacks.py +++ b/synapse/module_api/callbacks/account_validity_callbacks.py @@ -22,6 +22,7 @@ # Types for callbacks to be registered via the module api IS_USER_EXPIRED_CALLBACK = Callable[[str], Awaitable[Optional[bool]]] ON_USER_REGISTRATION_CALLBACK = Callable[[str], Awaitable] +ON_USER_LOGIN_CALLBACK = Callable[[str, Optional[str], Optional[str]], Awaitable] # Temporary hooks to allow for a transition from `/_matrix/client` endpoints # to `/_synapse/client/account_validity`. See `register_callbacks` below. ON_LEGACY_SEND_MAIL_CALLBACK = Callable[[str], Awaitable] @@ -33,6 +34,7 @@ class AccountValidityModuleApiCallbacks: def __init__(self) -> None: self.is_user_expired_callbacks: List[IS_USER_EXPIRED_CALLBACK] = [] self.on_user_registration_callbacks: List[ON_USER_REGISTRATION_CALLBACK] = [] + self.on_user_login_callbacks: List[ON_USER_LOGIN_CALLBACK] = [] self.on_legacy_send_mail_callback: Optional[ON_LEGACY_SEND_MAIL_CALLBACK] = None self.on_legacy_renew_callback: Optional[ON_LEGACY_RENEW_CALLBACK] = None @@ -44,6 +46,7 @@ def register_callbacks( self, is_user_expired: Optional[IS_USER_EXPIRED_CALLBACK] = None, on_user_registration: Optional[ON_USER_REGISTRATION_CALLBACK] = None, + on_user_login: Optional[ON_USER_LOGIN_CALLBACK] = None, on_legacy_send_mail: Optional[ON_LEGACY_SEND_MAIL_CALLBACK] = None, on_legacy_renew: Optional[ON_LEGACY_RENEW_CALLBACK] = None, on_legacy_admin_request: Optional[ON_LEGACY_ADMIN_REQUEST] = None, @@ -55,6 +58,9 @@ def register_callbacks( if on_user_registration is not None: self.on_user_registration_callbacks.append(on_user_registration) + if on_user_login is not None: + self.on_user_login_callbacks.append(on_user_login) + # The builtin account validity feature exposes 3 endpoints (send_mail, renew, and # an admin one). As part of moving the feature into a module, we need to change # the path from /_matrix/client/unstable/account_validity/... to diff --git a/synapse/rest/admin/federation.py b/synapse/rest/admin/federation.py index a6ce787da1af..2f238f021776 100644 --- a/synapse/rest/admin/federation.py +++ b/synapse/rest/admin/federation.py @@ -89,8 +89,8 @@ async def on_GET(self, request: SynapseRequest) -> Tuple[int, JsonDict]: "destinations": [ { "destination": r[0], - "retry_last_ts": r[1], - "retry_interval": r[2], + "retry_last_ts": r[1] or 0, + "retry_interval": r[2] or 0, "failure_ts": r[3], "last_successful_stream_ordering": r[4], } diff --git a/synapse/rest/client/login.py b/synapse/rest/client/login.py index 7be327e26f08..546f042f87c5 100644 --- a/synapse/rest/client/login.py +++ b/synapse/rest/client/login.py @@ -115,6 +115,7 @@ def __init__(self, hs: "HomeServer"): self.registration_handler = hs.get_registration_handler() self._sso_handler = hs.get_sso_handler() self._spam_checker = hs.get_module_api_callbacks().spam_checker + self._account_validity_handler = hs.get_account_validity_handler() self._well_known_builder = WellKnownBuilder(hs) self._address_ratelimiter = Ratelimiter( @@ -470,6 +471,13 @@ async def _complete_login( device_id=device_id, ) + # execute the callback + await self._account_validity_handler.on_user_login( + user_id, + auth_provider_type=login_submission.get("type"), + auth_provider_id=auth_provider_id, + ) + if valid_until_ms is not None: expires_in_ms = valid_until_ms - self.clock.time_msec() result["expires_in_ms"] = expires_in_ms diff --git a/synapse/rest/client/profile.py b/synapse/rest/client/profile.py index 493e1acea06e..12d3da1ced02 100644 --- a/synapse/rest/client/profile.py +++ b/synapse/rest/client/profile.py @@ -13,12 +13,17 @@ # limitations under the License. """ This module contains REST servlets to do with profile: /profile/ """ + from http import HTTPStatus from typing import TYPE_CHECKING, Tuple from synapse.api.errors import Codes, SynapseError from synapse.http.server import HttpServer -from synapse.http.servlet import RestServlet, parse_json_object_from_request +from synapse.http.servlet import ( + RestServlet, + parse_boolean, + parse_json_object_from_request, +) from synapse.http.site import SynapseRequest from synapse.rest.client._base import client_patterns from synapse.types import JsonDict, UserID @@ -27,6 +32,20 @@ from synapse.server import HomeServer +def _read_propagate(hs: "HomeServer", request: SynapseRequest) -> bool: + # This will always be set by the time Twisted calls us. + assert request.args is not None + + propagate = True + if hs.config.experimental.msc4069_profile_inhibit_propagation: + do_propagate = request.args.get(b"org.matrix.msc4069.propagate") + if do_propagate is not None: + propagate = parse_boolean( + request, "org.matrix.msc4069.propagate", default=False + ) + return propagate + + class ProfileDisplaynameRestServlet(RestServlet): PATTERNS = client_patterns("/profile/(?P[^/]*)/displayname", v1=True) CATEGORY = "Event sending requests" @@ -80,7 +99,11 @@ async def on_PUT( errcode=Codes.BAD_JSON, ) - await self.profile_handler.set_displayname(user, requester, new_name, is_admin) + propagate = _read_propagate(self.hs, request) + + await self.profile_handler.set_displayname( + user, requester, new_name, is_admin, propagate=propagate + ) return 200, {} @@ -135,8 +158,10 @@ async def on_PUT( 400, "Missing key 'avatar_url'", errcode=Codes.MISSING_PARAM ) + propagate = _read_propagate(self.hs, request) + await self.profile_handler.set_avatar_url( - user, requester, new_avatar_url, is_admin + user, requester, new_avatar_url, is_admin, propagate=propagate ) return 200, {} diff --git a/synapse/rest/client/versions.py b/synapse/rest/client/versions.py index 95400ba570a9..54c01bb739fc 100644 --- a/synapse/rest/client/versions.py +++ b/synapse/rest/client/versions.py @@ -80,6 +80,9 @@ def on_GET(self, request: Request) -> Tuple[int, JsonDict]: "v1.4", "v1.5", "v1.6", + "v1.7", + "v1.8", + "v1.9", ], # as per MSC1497: "unstable_features": { @@ -126,6 +129,8 @@ def on_GET(self, request: Request) -> Tuple[int, JsonDict]: "org.matrix.msc3981": self.config.experimental.msc3981_recurse_relations, # Adds support for deleting account data. "org.matrix.msc3391": self.config.experimental.msc3391_enabled, + # Allows clients to inhibit profile update propagation. + "org.matrix.msc4069": self.config.experimental.msc4069_profile_inhibit_propagation, }, }, ) diff --git a/synapse/server_notices/server_notices_manager.py b/synapse/server_notices/server_notices_manager.py index ebd89942443a..e7d5997b5c51 100644 --- a/synapse/server_notices/server_notices_manager.py +++ b/synapse/server_notices/server_notices_manager.py @@ -263,14 +263,27 @@ async def maybe_invite_user_to_room(self, user_id: str, room_id: str) -> None: if room.room_id == room_id: return + user_id_obj = UserID.from_string(user_id) await self._room_member_handler.update_membership( requester=requester, - target=UserID.from_string(user_id), + target=user_id_obj, room_id=room_id, action="invite", ratelimit=False, ) + if self._config.servernotices.server_notices_auto_join: + user_requester = create_requester( + user_id, authenticated_entity=self._server_name + ) + await self._room_member_handler.update_membership( + requester=user_requester, + target=user_id_obj, + room_id=room_id, + action="join", + ratelimit=False, + ) + async def _update_notice_user_profile_if_changed( self, requester: Requester, diff --git a/synapse/storage/databases/main/client_ips.py b/synapse/storage/databases/main/client_ips.py index d4b14aaebe7b..1df7731050d8 100644 --- a/synapse/storage/databases/main/client_ips.py +++ b/synapse/storage/databases/main/client_ips.py @@ -465,18 +465,15 @@ async def _prune_old_user_ips(self) -> None: # # This works by finding the max last_seen that is less than the given # time, but has no more than N rows before it, deleting all rows with - # a lesser last_seen time. (We COALESCE so that the sub-SELECT always - # returns exactly one row). + # a lesser last_seen time. (We use an `IN` clause to force postgres to + # use the index, otherwise it tends to do a seq scan). sql = """ DELETE FROM user_ips - WHERE last_seen <= ( - SELECT COALESCE(MAX(last_seen), -1) - FROM ( - SELECT last_seen FROM user_ips - WHERE last_seen <= ? - ORDER BY last_seen ASC - LIMIT 5000 - ) AS u + WHERE last_seen IN ( + SELECT last_seen FROM user_ips + WHERE last_seen <= ? + ORDER BY last_seen ASC + LIMIT 5000 ) """ diff --git a/synapse/storage/schema/__init__.py b/synapse/storage/schema/__init__.py index 03e5a0f55d3b..f87629b71933 100644 --- a/synapse/storage/schema/__init__.py +++ b/synapse/storage/schema/__init__.py @@ -129,8 +129,8 @@ SCHEMA_COMPAT_VERSION = ( - # The `event_txn_id_device_id` must be written to for new events. - 80 + # The event_txn_id table and tables from MSC2716 no longer exist. + 83 ) """Limit on how far the synapse codebase can be rolled back without breaking db compat diff --git a/synapse/storage/schema/main/delta/83/01_drop_old_tables.sql b/synapse/storage/schema/main/delta/83/01_drop_old_tables.sql new file mode 100644 index 000000000000..4b4dfe783340 --- /dev/null +++ b/synapse/storage/schema/main/delta/83/01_drop_old_tables.sql @@ -0,0 +1,24 @@ +/* Copyright 2023 The Matrix.org Foundation C.I.C + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +-- Drop the old event transaction ID table, the event_txn_id_device_id table +-- should be used instead. +DROP TABLE IF EXISTS event_txn_id; + +-- Drop tables related to MSC2716 since the implementation is being removed +DROP TABLE IF EXISTS insertion_events; +DROP TABLE IF EXISTS insertion_event_edges; +DROP TABLE IF EXISTS insertion_event_extremities; +DROP TABLE IF EXISTS batch_events; diff --git a/tests/media/test_media_storage.py b/tests/media/test_media_storage.py index f262304c3daa..f981d1c0d8dd 100644 --- a/tests/media/test_media_storage.py +++ b/tests/media/test_media_storage.py @@ -27,10 +27,11 @@ from twisted.internet import defer from twisted.internet.defer import Deferred +from twisted.python.failure import Failure from twisted.test.proto_helpers import MemoryReactor from twisted.web.resource import Resource -from synapse.api.errors import Codes +from synapse.api.errors import Codes, HttpResponseException from synapse.events import EventBase from synapse.http.types import QueryParams from synapse.logging.context import make_deferred_yieldable @@ -247,6 +248,7 @@ def get_file( retry_on_dns_fail: bool = True, max_size: Optional[int] = None, ignore_backoff: bool = False, + follow_redirects: bool = False, ) -> "Deferred[Tuple[int, Dict[bytes, List[bytes]]]]": """A mock for MatrixFederationHttpClient.get_file.""" @@ -257,10 +259,15 @@ def write_to( output_stream.write(data) return response + def write_err(f: Failure) -> Failure: + f.trap(HttpResponseException) + output_stream.write(f.value.response) + return f + d: Deferred[Tuple[bytes, Tuple[int, Dict[bytes, List[bytes]]]]] = Deferred() self.fetches.append((d, destination, path, args)) # Note that this callback changes the value held by d. - d_after_callback = d.addCallback(write_to) + d_after_callback = d.addCallbacks(write_to, write_err) return make_deferred_yieldable(d_after_callback) # Mock out the homeserver's MatrixFederationHttpClient @@ -316,10 +323,11 @@ def _req( self.assertEqual(len(self.fetches), 1) self.assertEqual(self.fetches[0][1], "example.com") self.assertEqual( - self.fetches[0][2], "/_matrix/media/r0/download/" + self.media_id + self.fetches[0][2], "/_matrix/media/v3/download/" + self.media_id ) self.assertEqual( - self.fetches[0][3], {"allow_remote": "false", "timeout_ms": "20000"} + self.fetches[0][3], + {"allow_remote": "false", "timeout_ms": "20000", "allow_redirect": "true"}, ) headers = { @@ -671,6 +679,52 @@ def test_cross_origin_resource_policy_header(self) -> None: [b"cross-origin"], ) + def test_unknown_v3_endpoint(self) -> None: + """ + If the v3 endpoint fails, try the r0 one. + """ + channel = self.make_request( + "GET", + f"/_matrix/media/v3/download/{self.media_id}", + shorthand=False, + await_result=False, + ) + self.pump() + + # We've made one fetch, to example.com, using the media URL, and asking + # the other server not to do a remote fetch + self.assertEqual(len(self.fetches), 1) + self.assertEqual(self.fetches[0][1], "example.com") + self.assertEqual( + self.fetches[0][2], "/_matrix/media/v3/download/" + self.media_id + ) + + # The result which says the endpoint is unknown. + unknown_endpoint = b'{"errcode":"M_UNRECOGNIZED","error":"Unknown request"}' + self.fetches[0][0].errback( + HttpResponseException(404, "NOT FOUND", unknown_endpoint) + ) + + self.pump() + + # There should now be another request to the r0 URL. + self.assertEqual(len(self.fetches), 2) + self.assertEqual(self.fetches[1][1], "example.com") + self.assertEqual( + self.fetches[1][2], f"/_matrix/media/r0/download/{self.media_id}" + ) + + headers = { + b"Content-Length": [b"%d" % (len(self.test_image.data))], + } + + self.fetches[1][0].callback( + (self.test_image.data, (len(self.test_image.data), headers)) + ) + + self.pump() + self.assertEqual(channel.code, 200) + class TestSpamCheckerLegacy: """A spam checker module that rejects all media that includes the bytes diff --git a/tests/replication/test_multi_media_repo.py b/tests/replication/test_multi_media_repo.py index 1e9994cc0bc7..9a7b675f54cb 100644 --- a/tests/replication/test_multi_media_repo.py +++ b/tests/replication/test_multi_media_repo.py @@ -133,7 +133,7 @@ def _get_media_req( self.assertEqual(request.method, b"GET") self.assertEqual( request.path, - f"/_matrix/media/r0/download/{target}/{media_id}".encode(), + f"/_matrix/media/v3/download/{target}/{media_id}".encode(), ) self.assertEqual( request.requestHeaders.getRawHeaders(b"host"), [target.encode("utf-8")] diff --git a/tests/rest/admin/test_server_notice.py b/tests/rest/admin/test_server_notice.py index 93b87b891b72..e1d4ceb69863 100644 --- a/tests/rest/admin/test_server_notice.py +++ b/tests/rest/admin/test_server_notice.py @@ -477,6 +477,33 @@ def test_send_server_notice_delete_room(self) -> None: # second room has new ID self.assertNotEqual(first_room_id, second_room_id) + @override_config( + {"server_notices": {"system_mxid_localpart": "notices", "auto_join": True}} + ) + def test_auto_join(self) -> None: + """ + Tests that the user get automatically joined to the notice room + when `auto_join` setting is used. + """ + # user has no room memberships + self._check_invite_and_join_status(self.other_user, 0, 0) + + # send server notice + server_notice_request_content = { + "user_id": self.other_user, + "content": {"msgtype": "m.text", "body": "test msg one"}, + } + + self.make_request( + "POST", + self.url, + access_token=self.admin_user_tok, + content=server_notice_request_content, + ) + + # user has joined the room + self._check_invite_and_join_status(self.other_user, 0, 1) + @override_config({"server_notices": {"system_mxid_localpart": "notices"}}) def test_update_notice_user_name_when_changed(self) -> None: """ diff --git a/tests/rest/client/test_profile.py b/tests/rest/client/test_profile.py index 8f923fd40f89..eb0fa00bb342 100644 --- a/tests/rest/client/test_profile.py +++ b/tests/rest/client/test_profile.py @@ -312,6 +312,166 @@ def test_avatar_allowed_mime_type_per_room(self) -> None: ) self.assertEqual(channel.code, 200, channel.result) + @unittest.override_config( + {"experimental_features": {"msc4069_profile_inhibit_propagation": True}} + ) + def test_msc4069_inhibit_propagation(self) -> None: + """Tests to ensure profile update propagation can be inhibited.""" + for prop in ["avatar_url", "displayname"]: + room_id = self.helper.create_room_as(tok=self.owner_tok) + + channel = self.make_request( + "PUT", + f"/rooms/{room_id}/state/m.room.member/{self.owner}", + content={"membership": "join", prop: "mxc://my.server/existing"}, + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + + channel = self.make_request( + "PUT", + f"/profile/{self.owner}/{prop}?org.matrix.msc4069.propagate=false", + content={prop: "http://my.server/pic.gif"}, + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + + res = ( + self._get_avatar_url() + if prop == "avatar_url" + else self._get_displayname() + ) + self.assertEqual(res, "http://my.server/pic.gif") + + channel = self.make_request( + "GET", + f"/rooms/{room_id}/state/m.room.member/{self.owner}", + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + self.assertEqual(channel.json_body.get(prop), "mxc://my.server/existing") + + def test_msc4069_inhibit_propagation_disabled(self) -> None: + """Tests to ensure profile update propagation inhibit flags are ignored when the + experimental flag is not enabled. + """ + for prop in ["avatar_url", "displayname"]: + room_id = self.helper.create_room_as(tok=self.owner_tok) + + channel = self.make_request( + "PUT", + f"/rooms/{room_id}/state/m.room.member/{self.owner}", + content={"membership": "join", prop: "mxc://my.server/existing"}, + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + + channel = self.make_request( + "PUT", + f"/profile/{self.owner}/{prop}?org.matrix.msc4069.propagate=false", + content={prop: "http://my.server/pic.gif"}, + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + + res = ( + self._get_avatar_url() + if prop == "avatar_url" + else self._get_displayname() + ) + self.assertEqual(res, "http://my.server/pic.gif") + + channel = self.make_request( + "GET", + f"/rooms/{room_id}/state/m.room.member/{self.owner}", + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + + # The ?propagate=false should be ignored by the server because the config flag + # isn't enabled. + self.assertEqual(channel.json_body.get(prop), "http://my.server/pic.gif") + + def test_msc4069_inhibit_propagation_default(self) -> None: + """Tests to ensure profile update propagation happens by default.""" + for prop in ["avatar_url", "displayname"]: + room_id = self.helper.create_room_as(tok=self.owner_tok) + + channel = self.make_request( + "PUT", + f"/rooms/{room_id}/state/m.room.member/{self.owner}", + content={"membership": "join", prop: "mxc://my.server/existing"}, + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + + channel = self.make_request( + "PUT", + f"/profile/{self.owner}/{prop}", + content={prop: "http://my.server/pic.gif"}, + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + + res = ( + self._get_avatar_url() + if prop == "avatar_url" + else self._get_displayname() + ) + self.assertEqual(res, "http://my.server/pic.gif") + + channel = self.make_request( + "GET", + f"/rooms/{room_id}/state/m.room.member/{self.owner}", + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + + # The ?propagate=false should be ignored by the server because the config flag + # isn't enabled. + self.assertEqual(channel.json_body.get(prop), "http://my.server/pic.gif") + + @unittest.override_config( + {"experimental_features": {"msc4069_profile_inhibit_propagation": True}} + ) + def test_msc4069_inhibit_propagation_like_default(self) -> None: + """Tests to ensure clients can request explicit profile propagation.""" + for prop in ["avatar_url", "displayname"]: + room_id = self.helper.create_room_as(tok=self.owner_tok) + + channel = self.make_request( + "PUT", + f"/rooms/{room_id}/state/m.room.member/{self.owner}", + content={"membership": "join", prop: "mxc://my.server/existing"}, + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + + channel = self.make_request( + "PUT", + f"/profile/{self.owner}/{prop}?org.matrix.msc4069.propagate=true", + content={prop: "http://my.server/pic.gif"}, + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + + res = ( + self._get_avatar_url() + if prop == "avatar_url" + else self._get_displayname() + ) + self.assertEqual(res, "http://my.server/pic.gif") + + channel = self.make_request( + "GET", + f"/rooms/{room_id}/state/m.room.member/{self.owner}", + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + + # The client requested ?propagate=true, so it should have happened. + self.assertEqual(channel.json_body.get(prop), "http://my.server/pic.gif") + def _setup_local_files(self, names_and_props: Dict[str, Dict[str, Any]]) -> None: """Stores metadata about files in the database. diff --git a/tests/rest/client/test_upgrade_room.py b/tests/rest/client/test_upgrade_room.py index 0b4c691318e9..28f772261b72 100644 --- a/tests/rest/client/test_upgrade_room.py +++ b/tests/rest/client/test_upgrade_room.py @@ -246,6 +246,34 @@ def test_stringy_power_levels(self) -> None: # We should now have an integer power level. self.assertEqual(new_power_levels["users"][self.creator], 100, new_power_levels) + def test_events_field_missing(self) -> None: + """Regression test for https://github.com/matrix-org/synapse/issues/16715.""" + # Create a new room. + room_id = self.helper.create_room_as( + self.creator, tok=self.creator_token, room_version="10" + ) + self.helper.join(room_id, self.other, tok=self.other_token) + + # Retrieve the room's current power levels. + power_levels = self.helper.get_state( + room_id, + "m.room.power_levels", + tok=self.creator_token, + ) + + # Remove the events field and re-set the power levels. + del power_levels["events"] + self.helper.send_state( + room_id, + "m.room.power_levels", + body=power_levels, + tok=self.creator_token, + ) + + # Upgrade the room. Check the homeserver reports success. + channel = self._upgrade_room(room_id=room_id) + self.assertEqual(200, channel.code, channel.result) + def test_space(self) -> None: """Test upgrading a space."""