From d75d6d65d1681889db05b077e97fc2ddf123b757 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 28 Nov 2023 14:09:21 +0000 Subject: [PATCH 01/29] 1.97.0 --- CHANGES.md | 8 +++++++- debian/changelog | 6 ++++++ pyproject.toml | 2 +- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 40a2e086c5c5..bb34267af331 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,4 +1,4 @@ -# Synapse 1.97.0rc1 (2023-11-21) +# Synapse 1.97.0 (2023-11-28) Synapse will soon be forked by Element under an AGPLv3.0 licence (with CLA, for proprietary dual licensing). You can read more about this here: @@ -10,6 +10,12 @@ The Matrix.org Foundation copy of the project will be archived. Any changes need by server administrators will be communicated via our usual announcements channels, but we are striving to make this as seamless as possible. + +No significant changes since 1.97.0rc1. + + +# Synapse 1.97.0rc1 (2023-11-21) + ### Features - Add support for asynchronous uploads as defined by [MSC2246](https://github.com/matrix-org/matrix-spec-proposals/pull/2246). Contributed by @sumnerevans at @beeper. ([\#15503](https://github.com/matrix-org/synapse/issues/15503)) diff --git a/debian/changelog b/debian/changelog index 1a4be59be1ce..5ec877160480 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.97.0) stable; urgency=medium + + * New Synapse release 1.97.0. + + -- Synapse Packaging team Tue, 28 Nov 2023 14:08:58 +0000 + matrix-synapse-py3 (1.97.0~rc1) stable; urgency=medium * New Synapse release 1.97.0rc1. diff --git a/pyproject.toml b/pyproject.toml index bd4ac09ae6da..189de0ca1ac1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -96,7 +96,7 @@ module-name = "synapse.synapse_rust" [tool.poetry] name = "matrix-synapse" -version = "1.97.0rc1" +version = "1.97.0" description = "Homeserver for the Matrix decentralised comms protocol" authors = ["Matrix.org Team and Contributors "] license = "Apache-2.0" From 77882b6a7d1ad1ab76b0ff878b3daed894bdb26e Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Tue, 28 Nov 2023 11:01:24 -0500 Subject: [PATCH 02/29] Document which versions of Synapse have compatible schema versions. (#16661) --- .github/workflows/docs-pr.yaml | 11 ++ .github/workflows/docs.yaml | 10 ++ book.toml | 5 +- changelog.d/16661.doc | 1 + docs/upgrade.md | 9 ++ scripts-dev/schema_versions.py | 181 +++++++++++++++++++++++++++++++++ 6 files changed, 216 insertions(+), 1 deletion(-) create mode 100644 changelog.d/16661.doc create mode 100755 scripts-dev/schema_versions.py diff --git a/.github/workflows/docs-pr.yaml b/.github/workflows/docs-pr.yaml index 3704bd66e2ce..9cf3d340a419 100644 --- a/.github/workflows/docs-pr.yaml +++ b/.github/workflows/docs-pr.yaml @@ -6,6 +6,7 @@ on: - docs/** - book.toml - .github/workflows/docs-pr.yaml + - scripts-dev/schema_versions.py jobs: pages: @@ -13,12 +14,22 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + with: + # Fetch all history so that the schema_versions script works. + fetch-depth: 0 - name: Setup mdbook uses: peaceiris/actions-mdbook@adeb05db28a0c0004681db83893d56c0388ea9ea # v1.2.0 with: mdbook-version: '0.4.17' + - name: Setup python + uses: actions/setup-python@v4 + with: + python-version: "3.x" + + - run: "pip install 'packaging>=20.0' 'GitPython>=3.1.20'" + - name: Build the documentation # mdbook will only create an index.html if we're including docs/README.md in SUMMARY.md. # However, we're using docs/README.md for other purposes and need to pick a new page diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index c7cb2d78e504..31b9dbe3fea6 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -51,12 +51,22 @@ jobs: - pre steps: - uses: actions/checkout@v4 + with: + # Fetch all history so that the schema_versions script works. + fetch-depth: 0 - name: Setup mdbook uses: peaceiris/actions-mdbook@adeb05db28a0c0004681db83893d56c0388ea9ea # v1.2.0 with: mdbook-version: '0.4.17' + - name: Setup python + uses: actions/setup-python@v4 + with: + python-version: "3.x" + + - run: "pip install 'packaging>=20.0' 'GitPython>=3.1.20'" + - name: Build the documentation # mdbook will only create an index.html if we're including docs/README.md in SUMMARY.md. # However, we're using docs/README.md for other purposes and need to pick a new page diff --git a/book.toml b/book.toml index fa83d86ffc1c..ed3f6151e06a 100644 --- a/book.toml +++ b/book.toml @@ -36,4 +36,7 @@ additional-css = [ "docs/website_files/indent-section-headers.css", ] additional-js = ["docs/website_files/table-of-contents.js"] -theme = "docs/website_files/theme" \ No newline at end of file +theme = "docs/website_files/theme" + +[preprocessor.schema_versions] +command = "./scripts-dev/schema_versions.py" diff --git a/changelog.d/16661.doc b/changelog.d/16661.doc new file mode 100644 index 000000000000..74f8fc84b8f7 --- /dev/null +++ b/changelog.d/16661.doc @@ -0,0 +1 @@ +Add schema rollback information to documentation. diff --git a/docs/upgrade.md b/docs/upgrade.md index ba2f7703bc75..329c9c778713 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -88,6 +88,15 @@ process, for example: dpkg -i matrix-synapse-py3_1.3.0+stretch1_amd64.deb ``` +Generally Synapse database schemas are compatible across multiple versions, once +a version of Synapse is deployed you may not be able to rollback automatically. +The following table gives the version ranges and the earliest version they can +be rolled back to. E.g. Synapse versions v1.58.0 through v1.61.1 can be rolled +back safely to v1.57.0, but starting with v1.62.0 it is only safe to rollback to +v1.61.0. + + + # Upgrading to v1.93.0 ## Minimum supported Rust version diff --git a/scripts-dev/schema_versions.py b/scripts-dev/schema_versions.py new file mode 100755 index 000000000000..5fd73251cdcb --- /dev/null +++ b/scripts-dev/schema_versions.py @@ -0,0 +1,181 @@ +#!/usr/bin/env python +# Copyright 2023 The Matrix.org Foundation C.I.C. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A script to calculate which versions of Synapse have backwards-compatible +database schemas. It creates a Markdown table of Synapse versions and the earliest +compatible version. + +It is compatible with the mdbook protocol for preprocessors (see +https://rust-lang.github.io/mdBook/for_developers/preprocessors.html#implementing-a-preprocessor-with-a-different-language): + +Exit 0 to denote support for all renderers: + + ./scripts-dev/schema_versions.py supports + +Parse a JSON list from stdin and add the table to the proper documetnation page: + + ./scripts-dev/schema_versions.py + +Additionally, the script supports dumping the table to stdout for debugging: + + ./scripts-dev/schema_versions.py dump +""" + +import io +import json +import sys +from collections import defaultdict +from typing import Any, Dict, Iterator, Optional, Tuple + +import git +from packaging import version + +# The schema version has moved around over the years. +SCHEMA_VERSION_FILES = ( + "synapse/storage/schema/__init__.py", + "synapse/storage/prepare_database.py", + "synapse/storage/__init__.py", + "synapse/app/homeserver.py", +) + + +# Skip versions of Synapse < v1.0, they're old and essentially not +# compatible with today's federation. +OLDEST_SHOWN_VERSION = version.parse("v1.0") + + +def get_schema_versions(tag: git.Tag) -> Tuple[Optional[int], Optional[int]]: + """Get the schema and schema compat versions for a tag.""" + schema_version = None + schema_compat_version = None + + for file in SCHEMA_VERSION_FILES: + try: + schema_file = tag.commit.tree / file + except KeyError: + continue + + # We (usually) can't execute the code since it might have unknown imports. + if file != "synapse/storage/schema/__init__.py": + with io.BytesIO(schema_file.data_stream.read()) as f: + for line in f.readlines(): + if line.startswith(b"SCHEMA_VERSION"): + schema_version = int(line.split()[2]) + + # Bail early. + break + else: + # SCHEMA_COMPAT_VERSION is sometimes across multiple lines, the easist + # thing to do is exec the code. Luckily it has only ever existed in + # a file which imports nothing else from Synapse. + locals: Dict[str, Any] = {} + exec(schema_file.data_stream.read().decode("utf-8"), {}, locals) + schema_version = locals["SCHEMA_VERSION"] + schema_compat_version = locals.get("SCHEMA_COMPAT_VERSION") + + return schema_version, schema_compat_version + + +def get_tags(repo: git.Repo) -> Iterator[git.Tag]: + """Return an iterator of tags sorted by version.""" + tags = [] + for tag in repo.tags: + # All "real" Synapse tags are of the form vX.Y.Z. + if not tag.name.startswith("v"): + continue + + # There's a weird tag from the initial react UI. + if tag.name == "v0.1": + continue + + try: + tag_version = version.parse(tag.name) + except version.InvalidVersion: + # Skip invalid versions. + continue + + # Skip pre- and post-release versions. + if tag_version.is_prerelease or tag_version.is_postrelease or tag_version.local: + continue + + # Skip old versions. + if tag_version < OLDEST_SHOWN_VERSION: + continue + + tags.append((tag_version, tag)) + + # Sort based on the version number (not lexically). + return (tag for _, tag in sorted(tags, key=lambda t: t[0])) + + +def calculate_version_chart() -> str: + repo = git.Repo(path=".") + + # Map of schema version -> Synapse versions which are at that schema version. + schema_versions = defaultdict(list) + # Map of schema version -> Synapse versions which are compatible with that + # schema version. + schema_compat_versions = defaultdict(list) + + # Find ranges of versions which are compatible with a schema version. + # + # There are two modes of operation: + # + # 1. Pre-schema_compat_version (i.e. schema_compat_version of None), then + # Synapse is compatible up/downgrading to a version with + # schema_version >= its current version. + # + # 2. Post-schema_compat_version (i.e. schema_compat_version is *not* None), + # then Synapse is compatible up/downgrading to a version with + # schema version >= schema_compat_version. + # + # This is more generous and avoids versions that cannot be rolled back. + # + # See https://github.com/matrix-org/synapse/pull/9933 which was included in v1.37.0. + for tag in get_tags(repo): + schema_version, schema_compat_version = get_schema_versions(tag) + + # If a schema compat version is given, prefer that over the schema version. + schema_versions[schema_version].append(tag.name) + schema_compat_versions[schema_compat_version or schema_version].append(tag.name) + + # Generate a table which maps the latest Synapse version compatible with each + # schema version. + result = f"| {'Versions': ^19} | Compatible version |\n" + result += f"|{'-' * (19 + 2)}|{'-' * (18 + 2)}|\n" + for schema_version, synapse_versions in schema_compat_versions.items(): + result += f"| {synapse_versions[0] + ' – ' + synapse_versions[-1]: ^19} | {schema_versions[schema_version][0]: ^18} |\n" + + return result + + +if __name__ == "__main__": + if len(sys.argv) == 3 and sys.argv[1] == "supports": + # We don't care about the renderer which is being used, which is the second argument. + sys.exit(0) + elif len(sys.argv) == 2 and sys.argv[1] == "dump": + print(calculate_version_chart()) + else: + # Expect JSON data on stdin. + context, book = json.load(sys.stdin) + + for section in book["sections"]: + if "Chapter" in section and section["Chapter"]["path"] == "upgrade.md": + section["Chapter"]["content"] = section["Chapter"]["content"].replace( + "", calculate_version_chart() + ) + + # Print the result back out to stdout. + print(json.dumps(book)) From 6f2be7794e9130ac52400c53ac6a04b557359733 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 28 Nov 2023 19:57:48 -0500 Subject: [PATCH 03/29] Bump cryptography from 41.0.5 to 41.0.6 (#16703) --- poetry.lock | 50 +++++++++++++++++++++++++------------------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/poetry.lock b/poetry.lock index c10167f25705..dd99f39ab723 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. [[package]] name = "alabaster" @@ -454,34 +454,34 @@ files = [ [[package]] name = "cryptography" -version = "41.0.5" +version = "41.0.6" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." optional = false python-versions = ">=3.7" files = [ - {file = "cryptography-41.0.5-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:da6a0ff8f1016ccc7477e6339e1d50ce5f59b88905585f77193ebd5068f1e797"}, - {file = "cryptography-41.0.5-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:b948e09fe5fb18517d99994184854ebd50b57248736fd4c720ad540560174ec5"}, - {file = "cryptography-41.0.5-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d38e6031e113b7421db1de0c1b1f7739564a88f1684c6b89234fbf6c11b75147"}, - {file = "cryptography-41.0.5-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e270c04f4d9b5671ebcc792b3ba5d4488bf7c42c3c241a3748e2599776f29696"}, - {file = "cryptography-41.0.5-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:ec3b055ff8f1dce8e6ef28f626e0972981475173d7973d63f271b29c8a2897da"}, - {file = "cryptography-41.0.5-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:7d208c21e47940369accfc9e85f0de7693d9a5d843c2509b3846b2db170dfd20"}, - {file = "cryptography-41.0.5-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:8254962e6ba1f4d2090c44daf50a547cd5f0bf446dc658a8e5f8156cae0d8548"}, - {file = "cryptography-41.0.5-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:a48e74dad1fb349f3dc1d449ed88e0017d792997a7ad2ec9587ed17405667e6d"}, - {file = "cryptography-41.0.5-cp37-abi3-win32.whl", hash = "sha256:d3977f0e276f6f5bf245c403156673db103283266601405376f075c849a0b936"}, - {file = "cryptography-41.0.5-cp37-abi3-win_amd64.whl", hash = "sha256:73801ac9736741f220e20435f84ecec75ed70eda90f781a148f1bad546963d81"}, - {file = "cryptography-41.0.5-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:3be3ca726e1572517d2bef99a818378bbcf7d7799d5372a46c79c29eb8d166c1"}, - {file = "cryptography-41.0.5-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:e886098619d3815e0ad5790c973afeee2c0e6e04b4da90b88e6bd06e2a0b1b72"}, - {file = "cryptography-41.0.5-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:573eb7128cbca75f9157dcde974781209463ce56b5804983e11a1c462f0f4e88"}, - {file = "cryptography-41.0.5-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:0c327cac00f082013c7c9fb6c46b7cc9fa3c288ca702c74773968173bda421bf"}, - {file = "cryptography-41.0.5-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:227ec057cd32a41c6651701abc0328135e472ed450f47c2766f23267b792a88e"}, - {file = "cryptography-41.0.5-pp38-pypy38_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:22892cc830d8b2c89ea60148227631bb96a7da0c1b722f2aac8824b1b7c0b6b8"}, - {file = "cryptography-41.0.5-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:5a70187954ba7292c7876734183e810b728b4f3965fbe571421cb2434d279179"}, - {file = "cryptography-41.0.5-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:88417bff20162f635f24f849ab182b092697922088b477a7abd6664ddd82291d"}, - {file = "cryptography-41.0.5-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:c707f7afd813478e2019ae32a7c49cd932dd60ab2d2a93e796f68236b7e1fbf1"}, - {file = "cryptography-41.0.5-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:580afc7b7216deeb87a098ef0674d6ee34ab55993140838b14c9b83312b37b86"}, - {file = "cryptography-41.0.5-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:fba1e91467c65fe64a82c689dc6cf58151158993b13eb7a7f3f4b7f395636723"}, - {file = "cryptography-41.0.5-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:0d2a6a598847c46e3e321a7aef8af1436f11c27f1254933746304ff014664d84"}, - {file = "cryptography-41.0.5.tar.gz", hash = "sha256:392cb88b597247177172e02da6b7a63deeff1937fa6fec3bbf902ebd75d97ec7"}, + {file = "cryptography-41.0.6-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:0f27acb55a4e77b9be8d550d762b0513ef3fc658cd3eb15110ebbcbd626db12c"}, + {file = "cryptography-41.0.6-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:ae236bb8760c1e55b7a39b6d4d32d2279bc6c7c8500b7d5a13b6fb9fc97be35b"}, + {file = "cryptography-41.0.6-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:afda76d84b053923c27ede5edc1ed7d53e3c9f475ebaf63c68e69f1403c405a8"}, + {file = "cryptography-41.0.6-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:da46e2b5df770070412c46f87bac0849b8d685c5f2679771de277a422c7d0b86"}, + {file = "cryptography-41.0.6-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:ff369dd19e8fe0528b02e8df9f2aeb2479f89b1270d90f96a63500afe9af5cae"}, + {file = "cryptography-41.0.6-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:b648fe2a45e426aaee684ddca2632f62ec4613ef362f4d681a9a6283d10e079d"}, + {file = "cryptography-41.0.6-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:5daeb18e7886a358064a68dbcaf441c036cbdb7da52ae744e7b9207b04d3908c"}, + {file = "cryptography-41.0.6-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:068bc551698c234742c40049e46840843f3d98ad7ce265fd2bd4ec0d11306596"}, + {file = "cryptography-41.0.6-cp37-abi3-win32.whl", hash = "sha256:2132d5865eea673fe6712c2ed5fb4fa49dba10768bb4cc798345748380ee3660"}, + {file = "cryptography-41.0.6-cp37-abi3-win_amd64.whl", hash = "sha256:48783b7e2bef51224020efb61b42704207dde583d7e371ef8fc2a5fb6c0aabc7"}, + {file = "cryptography-41.0.6-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:8efb2af8d4ba9dbc9c9dd8f04d19a7abb5b49eab1f3694e7b5a16a5fc2856f5c"}, + {file = "cryptography-41.0.6-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:c5a550dc7a3b50b116323e3d376241829fd326ac47bc195e04eb33a8170902a9"}, + {file = "cryptography-41.0.6-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:85abd057699b98fce40b41737afb234fef05c67e116f6f3650782c10862c43da"}, + {file = "cryptography-41.0.6-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:f39812f70fc5c71a15aa3c97b2bbe213c3f2a460b79bd21c40d033bb34a9bf36"}, + {file = "cryptography-41.0.6-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:742ae5e9a2310e9dade7932f9576606836ed174da3c7d26bc3d3ab4bd49b9f65"}, + {file = "cryptography-41.0.6-pp38-pypy38_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:35f3f288e83c3f6f10752467c48919a7a94b7d88cc00b0668372a0d2ad4f8ead"}, + {file = "cryptography-41.0.6-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:4d03186af98b1c01a4eda396b137f29e4e3fb0173e30f885e27acec8823c1b09"}, + {file = "cryptography-41.0.6-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:b27a7fd4229abef715e064269d98a7e2909ebf92eb6912a9603c7e14c181928c"}, + {file = "cryptography-41.0.6-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:398ae1fc711b5eb78e977daa3cbf47cec20f2c08c5da129b7a296055fbb22aed"}, + {file = "cryptography-41.0.6-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:7e00fb556bda398b99b0da289ce7053639d33b572847181d6483ad89835115f6"}, + {file = "cryptography-41.0.6-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:60e746b11b937911dc70d164060d28d273e31853bb359e2b2033c9e93e6f3c43"}, + {file = "cryptography-41.0.6-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:3288acccef021e3c3c10d58933f44e8602cf04dba96d9796d70d537bb2f4bbc4"}, + {file = "cryptography-41.0.6.tar.gz", hash = "sha256:422e3e31d63743855e43e5a6fcc8b4acab860f560f9321b0ee6269cc7ed70cc3"}, ] [package.dependencies] From df366966b4f16d22330f1a3783a6e4bee8aa22a7 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 29 Nov 2023 11:54:42 +0000 Subject: [PATCH 04/29] Speed up pruning of `user_ips` table (#16667) Silly query planner --- changelog.d/16667.misc | 1 + synapse/storage/databases/main/client_ips.py | 17 +++++++---------- 2 files changed, 8 insertions(+), 10 deletions(-) create mode 100644 changelog.d/16667.misc diff --git a/changelog.d/16667.misc b/changelog.d/16667.misc new file mode 100644 index 000000000000..51aeca924397 --- /dev/null +++ b/changelog.d/16667.misc @@ -0,0 +1 @@ +Reduce database load of pruning old `user_ips`. diff --git a/synapse/storage/databases/main/client_ips.py b/synapse/storage/databases/main/client_ips.py index d4b14aaebe7b..1df7731050d8 100644 --- a/synapse/storage/databases/main/client_ips.py +++ b/synapse/storage/databases/main/client_ips.py @@ -465,18 +465,15 @@ async def _prune_old_user_ips(self) -> None: # # This works by finding the max last_seen that is less than the given # time, but has no more than N rows before it, deleting all rows with - # a lesser last_seen time. (We COALESCE so that the sub-SELECT always - # returns exactly one row). + # a lesser last_seen time. (We use an `IN` clause to force postgres to + # use the index, otherwise it tends to do a seq scan). sql = """ DELETE FROM user_ips - WHERE last_seen <= ( - SELECT COALESCE(MAX(last_seen), -1) - FROM ( - SELECT last_seen FROM user_ips - WHERE last_seen <= ? - ORDER BY last_seen ASC - LIMIT 5000 - ) AS u + WHERE last_seen IN ( + SELECT last_seen FROM user_ips + WHERE last_seen <= ? + ORDER BY last_seen ASC + LIMIT 5000 ) """ From 19dac974808dce222ffdb2e8a2b380fb0856378c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 29 Nov 2023 14:07:32 +0000 Subject: [PATCH 05/29] Add a workflow to try and automatically fixup a PR (#16704) Co-authored-by: Patrick Cloke --- .github/workflows/fix_lint.yaml | 52 +++++++++++++++++++++++++++++++++ changelog.d/16704.misc | 1 + 2 files changed, 53 insertions(+) create mode 100644 .github/workflows/fix_lint.yaml create mode 100644 changelog.d/16704.misc diff --git a/.github/workflows/fix_lint.yaml b/.github/workflows/fix_lint.yaml new file mode 100644 index 000000000000..f1e35fcd99d9 --- /dev/null +++ b/.github/workflows/fix_lint.yaml @@ -0,0 +1,52 @@ +# A helper workflow to automatically fixup any linting errors on a PR. Must be +# triggered manually. + +name: Attempt to automatically fix linting errors + +on: + workflow_dispatch: + +jobs: + fixup: + name: Fix up + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Install Rust + uses: dtolnay/rust-toolchain@master + with: + # We use nightly so that `fmt` correctly groups together imports, and + # clippy correctly fixes up the benchmarks. + toolchain: nightly-2022-12-01 + components: rustfmt + - uses: Swatinem/rust-cache@v2 + + - name: Setup Poetry + uses: matrix-org/setup-python-poetry@v1 + with: + install-project: "false" + + - name: Import order (isort) + continue-on-error: true + run: poetry run isort . + + - name: Code style (black) + continue-on-error: true + run: poetry run black . + + - name: Semantic checks (ruff) + continue-on-error: true + run: poetry run ruff --fix . + + - run: cargo clippy --all-features --fix -- -D warnings + continue-on-error: true + + - run: cargo fmt + continue-on-error: true + + - uses: stefanzweifel/git-auto-commit-action@v5 + with: + commit_message: "Attempt to fix linting" diff --git a/changelog.d/16704.misc b/changelog.d/16704.misc new file mode 100644 index 000000000000..4dafb27fd8c1 --- /dev/null +++ b/changelog.d/16704.misc @@ -0,0 +1 @@ +Add a workflow to try and automatically fixup linting in a PR. From a14678492eed5482312000bd7423f765680e4afc Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 29 Nov 2023 18:21:30 +0000 Subject: [PATCH 06/29] Reduce DB load when forget on leave setting is disabled (#16668) * Reduce DB load when forget on leave setting is disabled * Newsfile --- changelog.d/16668.misc | 1 + synapse/handlers/room_member.py | 11 ++++++++--- 2 files changed, 9 insertions(+), 3 deletions(-) create mode 100644 changelog.d/16668.misc diff --git a/changelog.d/16668.misc b/changelog.d/16668.misc new file mode 100644 index 000000000000..9ed004d6e46a --- /dev/null +++ b/changelog.d/16668.misc @@ -0,0 +1 @@ +Reduce DB load when forget on leave setting is disabled. diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py index eddc2af9ba4c..00c2dd854d9f 100644 --- a/synapse/handlers/room_member.py +++ b/synapse/handlers/room_member.py @@ -2111,9 +2111,14 @@ async def _unsafe_process(self) -> None: self.pos = room_max_stream_ordering if not self._hs.config.room.forget_on_leave: - # Update the processing position, so that if the server admin turns the - # feature on at a later date, we don't decide to forget every room that - # has ever been left in the past. + # Update the processing position, so that if the server admin turns + # the feature on at a later date, we don't decide to forget every + # room that has ever been left in the past. + # + # We wait for a short time so that we don't "tight" loop just + # keeping the table up to date. + await self._clock.sleep(0.5) + self.pos = self._store.get_room_max_stream_ordering() await self._store.update_room_forgetter_stream_pos(self.pos) return From d6c3b7584fc46571e65226793304df35d7081534 Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Wed, 29 Nov 2023 14:03:42 -0500 Subject: [PATCH 07/29] Request & follow redirects for /media/v3/download (#16701) Implement MSC3860 to follow redirects for federated media downloads. Note that the Client-Server API doesn't support this (yet) since the media repository in Synapse doesn't have a way of supporting redirects. --- changelog.d/16701.feature | 1 + synapse/federation/federation_client.py | 38 +++++++++++ synapse/federation/transport/client.py | 53 +++++++++++++++ synapse/http/matrixfederationclient.py | 77 ++++++++++++++++------ synapse/media/media_repository.py | 17 ++--- tests/media/test_media_storage.py | 62 +++++++++++++++-- tests/replication/test_multi_media_repo.py | 2 +- 7 files changed, 212 insertions(+), 38 deletions(-) create mode 100644 changelog.d/16701.feature diff --git a/changelog.d/16701.feature b/changelog.d/16701.feature new file mode 100644 index 000000000000..2a66fc932adc --- /dev/null +++ b/changelog.d/16701.feature @@ -0,0 +1 @@ +Follow redirects when downloading media over federation (per [MSC3860](https://github.com/matrix-org/matrix-spec-proposals/pull/3860)). diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index 1a7fa175ec63..0ba03b0d0540 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -21,6 +21,7 @@ TYPE_CHECKING, AbstractSet, Awaitable, + BinaryIO, Callable, Collection, Container, @@ -1862,6 +1863,43 @@ def filter_user_id(user_id: str) -> bool: return filtered_statuses, filtered_failures + async def download_media( + self, + destination: str, + media_id: str, + output_stream: BinaryIO, + max_size: int, + max_timeout_ms: int, + ) -> Tuple[int, Dict[bytes, List[bytes]]]: + try: + return await self.transport_layer.download_media_v3( + destination, + media_id, + output_stream=output_stream, + max_size=max_size, + max_timeout_ms=max_timeout_ms, + ) + except HttpResponseException as e: + # If an error is received that is due to an unrecognised endpoint, + # fallback to the r0 endpoint. Otherwise, consider it a legitimate error + # and raise. + if not is_unknown_endpoint(e): + raise + + logger.debug( + "Couldn't download media %s/%s with the v3 API, falling back to the r0 API", + destination, + media_id, + ) + + return await self.transport_layer.download_media_r0( + destination, + media_id, + output_stream=output_stream, + max_size=max_size, + max_timeout_ms=max_timeout_ms, + ) + @attr.s(frozen=True, slots=True, auto_attribs=True) class TimestampToEventResponse: diff --git a/synapse/federation/transport/client.py b/synapse/federation/transport/client.py index fab480071716..5e36638b0a65 100644 --- a/synapse/federation/transport/client.py +++ b/synapse/federation/transport/client.py @@ -18,6 +18,7 @@ from typing import ( TYPE_CHECKING, Any, + BinaryIO, Callable, Collection, Dict, @@ -804,6 +805,58 @@ async def get_account_status( destination=destination, path=path, data={"user_ids": user_ids} ) + async def download_media_r0( + self, + destination: str, + media_id: str, + output_stream: BinaryIO, + max_size: int, + max_timeout_ms: int, + ) -> Tuple[int, Dict[bytes, List[bytes]]]: + path = f"/_matrix/media/r0/download/{destination}/{media_id}" + + return await self.client.get_file( + destination, + path, + output_stream=output_stream, + max_size=max_size, + args={ + # tell the remote server to 404 if it doesn't + # recognise the server_name, to make sure we don't + # end up with a routing loop. + "allow_remote": "false", + "timeout_ms": str(max_timeout_ms), + }, + ) + + async def download_media_v3( + self, + destination: str, + media_id: str, + output_stream: BinaryIO, + max_size: int, + max_timeout_ms: int, + ) -> Tuple[int, Dict[bytes, List[bytes]]]: + path = f"/_matrix/media/v3/download/{destination}/{media_id}" + + return await self.client.get_file( + destination, + path, + output_stream=output_stream, + max_size=max_size, + args={ + # tell the remote server to 404 if it doesn't + # recognise the server_name, to make sure we don't + # end up with a routing loop. + "allow_remote": "false", + "timeout_ms": str(max_timeout_ms), + # Matrix 1.7 allows for this to redirect to another URL, this should + # just be ignored for an old homeserver, so always provide it. + "allow_redirect": "true", + }, + follow_redirects=True, + ) + def _create_path(federation_prefix: str, path: str, *args: str) -> str: """ diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index d5013e8e97c1..cc1db763ae4e 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -153,12 +153,18 @@ class MatrixFederationRequest: """Query arguments. """ - txn_id: Optional[str] = None - """Unique ID for this request (for logging) + txn_id: str = attr.ib(init=False) + """Unique ID for this request (for logging), this is autogenerated. """ - uri: bytes = attr.ib(init=False) - """The URI of this request + uri: bytes = b"" + """The URI of this request, usually generated from the above information. + """ + + _generate_uri: bool = True + """True to automatically generate the uri field based on the above information. + + Set to False if manually configuring the URI. """ def __attrs_post_init__(self) -> None: @@ -168,22 +174,23 @@ def __attrs_post_init__(self) -> None: object.__setattr__(self, "txn_id", txn_id) - destination_bytes = self.destination.encode("ascii") - path_bytes = self.path.encode("ascii") - query_bytes = encode_query_args(self.query) - - # The object is frozen so we can pre-compute this. - uri = urllib.parse.urlunparse( - ( - b"matrix-federation", - destination_bytes, - path_bytes, - None, - query_bytes, - b"", + if self._generate_uri: + destination_bytes = self.destination.encode("ascii") + path_bytes = self.path.encode("ascii") + query_bytes = encode_query_args(self.query) + + # The object is frozen so we can pre-compute this. + uri = urllib.parse.urlunparse( + ( + b"matrix-federation", + destination_bytes, + path_bytes, + None, + query_bytes, + b"", + ) ) - ) - object.__setattr__(self, "uri", uri) + object.__setattr__(self, "uri", uri) def get_json(self) -> Optional[JsonDict]: if self.json_callback: @@ -513,6 +520,7 @@ async def _send_request( ignore_backoff: bool = False, backoff_on_404: bool = False, backoff_on_all_error_codes: bool = False, + follow_redirects: bool = False, ) -> IResponse: """ Sends a request to the given server. @@ -555,6 +563,9 @@ async def _send_request( backoff_on_404: Back off if we get a 404 backoff_on_all_error_codes: Back off if we get any error response + follow_redirects: True to follow the Location header of 307/308 redirect + responses. This does not recurse. + Returns: Resolves with the HTTP response object on success. @@ -714,6 +725,26 @@ async def _send_request( response.code, response_phrase, ) + elif ( + response.code in (307, 308) + and follow_redirects + and response.headers.hasHeader("Location") + ): + # The Location header *might* be relative so resolve it. + location = response.headers.getRawHeaders(b"Location")[0] + new_uri = urllib.parse.urljoin(request.uri, location) + + return await self._send_request( + attr.evolve(request, uri=new_uri, generate_uri=False), + retry_on_dns_fail, + timeout, + long_retries, + ignore_backoff, + backoff_on_404, + backoff_on_all_error_codes, + # Do not continue following redirects. + follow_redirects=False, + ) else: logger.info( "{%s} [%s] Got response headers: %d %s", @@ -1383,6 +1414,7 @@ async def get_file( retry_on_dns_fail: bool = True, max_size: Optional[int] = None, ignore_backoff: bool = False, + follow_redirects: bool = False, ) -> Tuple[int, Dict[bytes, List[bytes]]]: """GETs a file from a given homeserver Args: @@ -1392,6 +1424,8 @@ async def get_file( args: Optional dictionary used to create the query string. ignore_backoff: true to ignore the historical backoff data and try the request anyway. + follow_redirects: True to follow the Location header of 307/308 redirect + responses. This does not recurse. Returns: Resolves with an (int,dict) tuple of @@ -1412,7 +1446,10 @@ async def get_file( ) response = await self._send_request( - request, retry_on_dns_fail=retry_on_dns_fail, ignore_backoff=ignore_backoff + request, + retry_on_dns_fail=retry_on_dns_fail, + ignore_backoff=ignore_backoff, + follow_redirects=follow_redirects, ) headers = dict(response.headers.getAllRawHeaders()) diff --git a/synapse/media/media_repository.py b/synapse/media/media_repository.py index bf976b9e7c2c..d62af22adb70 100644 --- a/synapse/media/media_repository.py +++ b/synapse/media/media_repository.py @@ -77,7 +77,7 @@ class MediaRepository: def __init__(self, hs: "HomeServer"): self.hs = hs self.auth = hs.get_auth() - self.client = hs.get_federation_http_client() + self.client = hs.get_federation_client() self.clock = hs.get_clock() self.server_name = hs.hostname self.store = hs.get_datastores().main @@ -644,22 +644,13 @@ async def _download_remote_file( file_info = FileInfo(server_name=server_name, file_id=file_id) with self.media_storage.store_into_file(file_info) as (f, fname, finish): - request_path = "/".join( - ("/_matrix/media/r0/download", server_name, media_id) - ) try: - length, headers = await self.client.get_file( + length, headers = await self.client.download_media( server_name, - request_path, + media_id, output_stream=f, max_size=self.max_upload_size, - args={ - # tell the remote server to 404 if it doesn't - # recognise the server_name, to make sure we don't - # end up with a routing loop. - "allow_remote": "false", - "timeout_ms": str(max_timeout_ms), - }, + max_timeout_ms=max_timeout_ms, ) except RequestSendFailed as e: logger.warning( diff --git a/tests/media/test_media_storage.py b/tests/media/test_media_storage.py index f262304c3daa..f981d1c0d8dd 100644 --- a/tests/media/test_media_storage.py +++ b/tests/media/test_media_storage.py @@ -27,10 +27,11 @@ from twisted.internet import defer from twisted.internet.defer import Deferred +from twisted.python.failure import Failure from twisted.test.proto_helpers import MemoryReactor from twisted.web.resource import Resource -from synapse.api.errors import Codes +from synapse.api.errors import Codes, HttpResponseException from synapse.events import EventBase from synapse.http.types import QueryParams from synapse.logging.context import make_deferred_yieldable @@ -247,6 +248,7 @@ def get_file( retry_on_dns_fail: bool = True, max_size: Optional[int] = None, ignore_backoff: bool = False, + follow_redirects: bool = False, ) -> "Deferred[Tuple[int, Dict[bytes, List[bytes]]]]": """A mock for MatrixFederationHttpClient.get_file.""" @@ -257,10 +259,15 @@ def write_to( output_stream.write(data) return response + def write_err(f: Failure) -> Failure: + f.trap(HttpResponseException) + output_stream.write(f.value.response) + return f + d: Deferred[Tuple[bytes, Tuple[int, Dict[bytes, List[bytes]]]]] = Deferred() self.fetches.append((d, destination, path, args)) # Note that this callback changes the value held by d. - d_after_callback = d.addCallback(write_to) + d_after_callback = d.addCallbacks(write_to, write_err) return make_deferred_yieldable(d_after_callback) # Mock out the homeserver's MatrixFederationHttpClient @@ -316,10 +323,11 @@ def _req( self.assertEqual(len(self.fetches), 1) self.assertEqual(self.fetches[0][1], "example.com") self.assertEqual( - self.fetches[0][2], "/_matrix/media/r0/download/" + self.media_id + self.fetches[0][2], "/_matrix/media/v3/download/" + self.media_id ) self.assertEqual( - self.fetches[0][3], {"allow_remote": "false", "timeout_ms": "20000"} + self.fetches[0][3], + {"allow_remote": "false", "timeout_ms": "20000", "allow_redirect": "true"}, ) headers = { @@ -671,6 +679,52 @@ def test_cross_origin_resource_policy_header(self) -> None: [b"cross-origin"], ) + def test_unknown_v3_endpoint(self) -> None: + """ + If the v3 endpoint fails, try the r0 one. + """ + channel = self.make_request( + "GET", + f"/_matrix/media/v3/download/{self.media_id}", + shorthand=False, + await_result=False, + ) + self.pump() + + # We've made one fetch, to example.com, using the media URL, and asking + # the other server not to do a remote fetch + self.assertEqual(len(self.fetches), 1) + self.assertEqual(self.fetches[0][1], "example.com") + self.assertEqual( + self.fetches[0][2], "/_matrix/media/v3/download/" + self.media_id + ) + + # The result which says the endpoint is unknown. + unknown_endpoint = b'{"errcode":"M_UNRECOGNIZED","error":"Unknown request"}' + self.fetches[0][0].errback( + HttpResponseException(404, "NOT FOUND", unknown_endpoint) + ) + + self.pump() + + # There should now be another request to the r0 URL. + self.assertEqual(len(self.fetches), 2) + self.assertEqual(self.fetches[1][1], "example.com") + self.assertEqual( + self.fetches[1][2], f"/_matrix/media/r0/download/{self.media_id}" + ) + + headers = { + b"Content-Length": [b"%d" % (len(self.test_image.data))], + } + + self.fetches[1][0].callback( + (self.test_image.data, (len(self.test_image.data), headers)) + ) + + self.pump() + self.assertEqual(channel.code, 200) + class TestSpamCheckerLegacy: """A spam checker module that rejects all media that includes the bytes diff --git a/tests/replication/test_multi_media_repo.py b/tests/replication/test_multi_media_repo.py index 1e9994cc0bc7..9a7b675f54cb 100644 --- a/tests/replication/test_multi_media_repo.py +++ b/tests/replication/test_multi_media_repo.py @@ -133,7 +133,7 @@ def _get_media_req( self.assertEqual(request.method, b"GET") self.assertEqual( request.path, - f"/_matrix/media/r0/download/{target}/{media_id}".encode(), + f"/_matrix/media/v3/download/{target}/{media_id}".encode(), ) self.assertEqual( request.requestHeaders.getRawHeaders(b"host"), [target.encode("utf-8")] From dcf949cd879c353340b906330b61a9b5c849f69a Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Wed, 29 Nov 2023 15:02:09 -0500 Subject: [PATCH 08/29] Declare support for Matrix v1.7, v1.8, and v1.9. (#16707) --- changelog.d/16707.feature | 1 + synapse/rest/client/versions.py | 3 +++ 2 files changed, 4 insertions(+) create mode 100644 changelog.d/16707.feature diff --git a/changelog.d/16707.feature b/changelog.d/16707.feature new file mode 100644 index 000000000000..707502d344d5 --- /dev/null +++ b/changelog.d/16707.feature @@ -0,0 +1 @@ +Synapse now declares support for Matrix v1.7, v1.8, and v1.9. diff --git a/synapse/rest/client/versions.py b/synapse/rest/client/versions.py index 95400ba570a9..f4d19e04706d 100644 --- a/synapse/rest/client/versions.py +++ b/synapse/rest/client/versions.py @@ -80,6 +80,9 @@ def on_GET(self, request: Request) -> Tuple[int, JsonDict]: "v1.4", "v1.5", "v1.6", + "v1.7", + "v1.8", + "v1.9", ], # as per MSC1497: "unstable_features": { From 3a092699e569ebec37b0624be8164238cef949a5 Mon Sep 17 00:00:00 2001 From: Mo Balaa Date: Wed, 29 Nov 2023 14:46:43 -0600 Subject: [PATCH 09/29] Upgrade poetry-core range to fix issue with .so file (#16702) poetry-core 1.8.x includes a fix which properly moves the generate synapse_rust.abi3.so file to the synapse directory when using an editable install. Without this change developers are left with a confusing experience of the synapse.synapse_rust module not being found after installation. --- changelog.d/16702.misc | 1 + pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/16702.misc diff --git a/changelog.d/16702.misc b/changelog.d/16702.misc new file mode 100644 index 000000000000..d97d0a11be76 --- /dev/null +++ b/changelog.d/16702.misc @@ -0,0 +1 @@ +Raise poetry-core upper bound to <=1.8.1. This allows contributors to import Synapse after `poetry install`ing with Poetry 1.6 and above. Contributed by Mo Balaa. diff --git a/pyproject.toml b/pyproject.toml index b2a4fd7a2fe6..b0ccf82e1087 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -382,7 +382,7 @@ furo = ">=2022.12.7,<2024.0.0" # runtime errors caused by build system changes. # We are happy to raise these upper bounds upon request, # provided we check that it's safe to do so (i.e. that CI passes). -requires = ["poetry-core>=1.1.0,<=1.7.0", "setuptools_rust>=1.3,<=1.8.1"] +requires = ["poetry-core>=1.1.0,<=1.8.1", "setuptools_rust>=1.3,<=1.8.1"] build-backend = "poetry.core.masonry.api" From 579c6be5f6eabcb05061730ea4d04d000e105076 Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Fri, 1 Dec 2023 05:12:00 -0500 Subject: [PATCH 10/29] Drop unused tables & unneeded access token ID for events. (#16522) --- changelog.d/16522.misc | 1 + synapse/handlers/message.py | 8 ++----- synapse/storage/schema/__init__.py | 4 ++-- .../main/delta/83/01_drop_old_tables.sql | 24 +++++++++++++++++++ 4 files changed, 29 insertions(+), 8 deletions(-) create mode 100644 changelog.d/16522.misc create mode 100644 synapse/storage/schema/main/delta/83/01_drop_old_tables.sql diff --git a/changelog.d/16522.misc b/changelog.d/16522.misc new file mode 100644 index 000000000000..26059b108e6a --- /dev/null +++ b/changelog.d/16522.misc @@ -0,0 +1 @@ +Clean-up unused tables. diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 811a41f161b6..25dd96416a66 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -693,13 +693,9 @@ async def create_event( if require_consent and not is_exempt: await self.assert_accepted_privacy_policy(requester) - # Save the access token ID, the device ID and the transaction ID in the event - # internal metadata. This is useful to determine if we should echo the - # transaction_id in events. + # Save the the device ID and the transaction ID in the event internal metadata. + # This is useful to determine if we should echo the transaction_id in events. # See `synapse.events.utils.EventClientSerializer.serialize_event` - if requester.access_token_id is not None: - builder.internal_metadata.token_id = requester.access_token_id - if requester.device_id is not None: builder.internal_metadata.device_id = requester.device_id diff --git a/synapse/storage/schema/__init__.py b/synapse/storage/schema/__init__.py index 03e5a0f55d3b..f87629b71933 100644 --- a/synapse/storage/schema/__init__.py +++ b/synapse/storage/schema/__init__.py @@ -129,8 +129,8 @@ SCHEMA_COMPAT_VERSION = ( - # The `event_txn_id_device_id` must be written to for new events. - 80 + # The event_txn_id table and tables from MSC2716 no longer exist. + 83 ) """Limit on how far the synapse codebase can be rolled back without breaking db compat diff --git a/synapse/storage/schema/main/delta/83/01_drop_old_tables.sql b/synapse/storage/schema/main/delta/83/01_drop_old_tables.sql new file mode 100644 index 000000000000..4b4dfe783340 --- /dev/null +++ b/synapse/storage/schema/main/delta/83/01_drop_old_tables.sql @@ -0,0 +1,24 @@ +/* Copyright 2023 The Matrix.org Foundation C.I.C + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +-- Drop the old event transaction ID table, the event_txn_id_device_id table +-- should be used instead. +DROP TABLE IF EXISTS event_txn_id; + +-- Drop tables related to MSC2716 since the implementation is being removed +DROP TABLE IF EXISTS insertion_events; +DROP TABLE IF EXISTS insertion_event_edges; +DROP TABLE IF EXISTS insertion_event_extremities; +DROP TABLE IF EXISTS batch_events; From 63d96bfc61fcbf53e9607c63f215d2dde387de29 Mon Sep 17 00:00:00 2001 From: Andrew Yasinishyn Date: Fri, 1 Dec 2023 16:31:50 +0200 Subject: [PATCH 11/29] ModuleAPI SSO auth callbacks (#15207) Signed-off-by: Andrii Yasynyshyn yasinishyn.a.n@gmail.com --- changelog.d/15207.feature | 1 + docs/modules/account_validity_callbacks.md | 13 +++++++++++++ rust/src/push/mod.rs | 3 +-- synapse/handlers/account_validity.py | 16 ++++++++++++++++ synapse/handlers/auth.py | 8 ++++++++ synapse/module_api/__init__.py | 3 +++ .../callbacks/account_validity_callbacks.py | 6 ++++++ synapse/rest/client/login.py | 8 ++++++++ 8 files changed, 56 insertions(+), 2 deletions(-) create mode 100644 changelog.d/15207.feature diff --git a/changelog.d/15207.feature b/changelog.d/15207.feature new file mode 100644 index 000000000000..17790d62eb09 --- /dev/null +++ b/changelog.d/15207.feature @@ -0,0 +1 @@ +Adds on_user_login ModuleAPI callback allowing to execute custom code after (on) Auth. \ No newline at end of file diff --git a/docs/modules/account_validity_callbacks.md b/docs/modules/account_validity_callbacks.md index 3cd0e7219894..f5eefcd7d639 100644 --- a/docs/modules/account_validity_callbacks.md +++ b/docs/modules/account_validity_callbacks.md @@ -42,3 +42,16 @@ operations to keep track of them. (e.g. add them to a database table). The user represented by their Matrix user ID. If multiple modules implement this callback, Synapse runs them all in order. + +### `on_user_login` + +_First introduced in Synapse v1.98.0_ + +```python +async def on_user_login(user_id: str, auth_provider_type: str, auth_provider_id: str) -> None +``` + +Called after successfully login or registration of a user for cases when module needs to perform extra operations after auth. +represented by their Matrix user ID. + +If multiple modules implement this callback, Synapse runs them all in order. diff --git a/rust/src/push/mod.rs b/rust/src/push/mod.rs index 5e1e8e1abbec..68d4227baa81 100644 --- a/rust/src/push/mod.rs +++ b/rust/src/push/mod.rs @@ -296,8 +296,7 @@ impl<'source> FromPyObject<'source> for JsonValue { match l.iter().map(SimpleJsonValue::extract).collect() { Ok(a) => Ok(JsonValue::Array(a)), Err(e) => Err(PyTypeError::new_err(format!( - "Can't convert to JsonValue::Array: {}", - e + "Can't convert to JsonValue::Array: {e}" ))), } } else if let Ok(v) = SimpleJsonValue::extract(ob) { diff --git a/synapse/handlers/account_validity.py b/synapse/handlers/account_validity.py index 6c2a49a3b91f..c66bb6364f08 100644 --- a/synapse/handlers/account_validity.py +++ b/synapse/handlers/account_validity.py @@ -98,6 +98,22 @@ async def on_user_registration(self, user_id: str) -> None: for callback in self._module_api_callbacks.on_user_registration_callbacks: await callback(user_id) + async def on_user_login( + self, + user_id: str, + auth_provider_type: Optional[str], + auth_provider_id: Optional[str], + ) -> None: + """Tell third-party modules about a user logins. + + Args: + user_id: The mxID of the user. + auth_provider_type: The type of login. + auth_provider_id: The ID of the auth provider. + """ + for callback in self._module_api_callbacks.on_user_login_callbacks: + await callback(user_id, auth_provider_type, auth_provider_id) + @wrap_as_background_process("send_renewals") async def _send_renewal_emails(self) -> None: """Gets the list of users whose account is expiring in the amount of time diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index 2b0c50513095..89cbaff864d4 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -212,6 +212,7 @@ def __init__(self, hs: "HomeServer"): self._password_enabled_for_reauth = hs.config.auth.password_enabled_for_reauth self._password_localdb_enabled = hs.config.auth.password_localdb_enabled self._third_party_rules = hs.get_module_api_callbacks().third_party_event_rules + self._account_validity_handler = hs.get_account_validity_handler() # Ratelimiter for failed auth during UIA. Uses same ratelimit config # as per `rc_login.failed_attempts`. @@ -1783,6 +1784,13 @@ async def complete_sso_login( client_redirect_url, "loginToken", login_token ) + # Run post-login module callback handlers + await self._account_validity_handler.on_user_login( + user_id=registered_user_id, + auth_provider_type=LoginType.SSO, + auth_provider_id=auth_provider_id, + ) + # if the client is whitelisted, we can redirect straight to it if client_redirect_url.startswith(self._whitelisted_sso_clients): request.redirect(redirect_url) diff --git a/synapse/module_api/__init__.py b/synapse/module_api/__init__.py index 812144a128a0..6ee53511f2f6 100644 --- a/synapse/module_api/__init__.py +++ b/synapse/module_api/__init__.py @@ -80,6 +80,7 @@ ON_LEGACY_ADMIN_REQUEST, ON_LEGACY_RENEW_CALLBACK, ON_LEGACY_SEND_MAIL_CALLBACK, + ON_USER_LOGIN_CALLBACK, ON_USER_REGISTRATION_CALLBACK, ) from synapse.module_api.callbacks.spamchecker_callbacks import ( @@ -334,6 +335,7 @@ def register_account_validity_callbacks( *, is_user_expired: Optional[IS_USER_EXPIRED_CALLBACK] = None, on_user_registration: Optional[ON_USER_REGISTRATION_CALLBACK] = None, + on_user_login: Optional[ON_USER_LOGIN_CALLBACK] = None, on_legacy_send_mail: Optional[ON_LEGACY_SEND_MAIL_CALLBACK] = None, on_legacy_renew: Optional[ON_LEGACY_RENEW_CALLBACK] = None, on_legacy_admin_request: Optional[ON_LEGACY_ADMIN_REQUEST] = None, @@ -345,6 +347,7 @@ def register_account_validity_callbacks( return self._callbacks.account_validity.register_callbacks( is_user_expired=is_user_expired, on_user_registration=on_user_registration, + on_user_login=on_user_login, on_legacy_send_mail=on_legacy_send_mail, on_legacy_renew=on_legacy_renew, on_legacy_admin_request=on_legacy_admin_request, diff --git a/synapse/module_api/callbacks/account_validity_callbacks.py b/synapse/module_api/callbacks/account_validity_callbacks.py index 531d0c9ddcf4..cbfdfbd3d186 100644 --- a/synapse/module_api/callbacks/account_validity_callbacks.py +++ b/synapse/module_api/callbacks/account_validity_callbacks.py @@ -22,6 +22,7 @@ # Types for callbacks to be registered via the module api IS_USER_EXPIRED_CALLBACK = Callable[[str], Awaitable[Optional[bool]]] ON_USER_REGISTRATION_CALLBACK = Callable[[str], Awaitable] +ON_USER_LOGIN_CALLBACK = Callable[[str, Optional[str], Optional[str]], Awaitable] # Temporary hooks to allow for a transition from `/_matrix/client` endpoints # to `/_synapse/client/account_validity`. See `register_callbacks` below. ON_LEGACY_SEND_MAIL_CALLBACK = Callable[[str], Awaitable] @@ -33,6 +34,7 @@ class AccountValidityModuleApiCallbacks: def __init__(self) -> None: self.is_user_expired_callbacks: List[IS_USER_EXPIRED_CALLBACK] = [] self.on_user_registration_callbacks: List[ON_USER_REGISTRATION_CALLBACK] = [] + self.on_user_login_callbacks: List[ON_USER_LOGIN_CALLBACK] = [] self.on_legacy_send_mail_callback: Optional[ON_LEGACY_SEND_MAIL_CALLBACK] = None self.on_legacy_renew_callback: Optional[ON_LEGACY_RENEW_CALLBACK] = None @@ -44,6 +46,7 @@ def register_callbacks( self, is_user_expired: Optional[IS_USER_EXPIRED_CALLBACK] = None, on_user_registration: Optional[ON_USER_REGISTRATION_CALLBACK] = None, + on_user_login: Optional[ON_USER_LOGIN_CALLBACK] = None, on_legacy_send_mail: Optional[ON_LEGACY_SEND_MAIL_CALLBACK] = None, on_legacy_renew: Optional[ON_LEGACY_RENEW_CALLBACK] = None, on_legacy_admin_request: Optional[ON_LEGACY_ADMIN_REQUEST] = None, @@ -55,6 +58,9 @@ def register_callbacks( if on_user_registration is not None: self.on_user_registration_callbacks.append(on_user_registration) + if on_user_login is not None: + self.on_user_login_callbacks.append(on_user_login) + # The builtin account validity feature exposes 3 endpoints (send_mail, renew, and # an admin one). As part of moving the feature into a module, we need to change # the path from /_matrix/client/unstable/account_validity/... to diff --git a/synapse/rest/client/login.py b/synapse/rest/client/login.py index 7be327e26f08..546f042f87c5 100644 --- a/synapse/rest/client/login.py +++ b/synapse/rest/client/login.py @@ -115,6 +115,7 @@ def __init__(self, hs: "HomeServer"): self.registration_handler = hs.get_registration_handler() self._sso_handler = hs.get_sso_handler() self._spam_checker = hs.get_module_api_callbacks().spam_checker + self._account_validity_handler = hs.get_account_validity_handler() self._well_known_builder = WellKnownBuilder(hs) self._address_ratelimiter = Ratelimiter( @@ -470,6 +471,13 @@ async def _complete_login( device_id=device_id, ) + # execute the callback + await self._account_validity_handler.on_user_login( + user_id, + auth_provider_type=login_submission.get("type"), + auth_provider_id=auth_provider_id, + ) + if valid_until_ms is not None: expires_in_ms = valid_until_ms - self.clock.time_msec() result["expires_in_ms"] = expires_in_ms From a5c14346fab8e6da8094f8d54e1cdb8488a13d87 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Dec 2023 11:16:45 +0000 Subject: [PATCH 12/29] Bump types-jsonschema from 4.19.0.4 to 4.20.0.0 (#16724) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index dd99f39ab723..9d47091e2ccb 100644 --- a/poetry.lock +++ b/poetry.lock @@ -3054,13 +3054,13 @@ files = [ [[package]] name = "types-jsonschema" -version = "4.19.0.4" +version = "4.20.0.0" description = "Typing stubs for jsonschema" optional = false python-versions = ">=3.8" files = [ - {file = "types-jsonschema-4.19.0.4.tar.gz", hash = "sha256:994feb6632818259c4b5dbd733867824cb475029a6abc2c2b5201a2268b6e7d2"}, - {file = "types_jsonschema-4.19.0.4-py3-none-any.whl", hash = "sha256:b73c3f4ba3cd8108602d1198a438e2698d5eb6b9db206ed89a33e24729b0abe7"}, + {file = "types-jsonschema-4.20.0.0.tar.gz", hash = "sha256:0de1032d243f1d3dba8b745ad84efe8c1af71665a9deb1827636ac535dcb79c1"}, + {file = "types_jsonschema-4.20.0.0-py3-none-any.whl", hash = "sha256:e6d5df18aaca4412f0aae246a294761a92040e93d7bc840f002b7329a8b72d26"}, ] [package.dependencies] From dd02c6340e3501fb284a8d8328c30f24186a81ea Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Dec 2023 11:17:42 +0000 Subject: [PATCH 13/29] Bump sphinx-autodoc2 from 0.4.2 to 0.5.0 (#16723) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 12 ++++++------ pyproject.toml | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/poetry.lock b/poetry.lock index 9d47091e2ccb..c2b8f8f43766 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2691,17 +2691,17 @@ test = ["cython", "filelock", "html5lib", "pytest (>=4.6)"] [[package]] name = "sphinx-autodoc2" -version = "0.4.2" +version = "0.5.0" description = "Analyse a python project and create documentation for it." optional = false python-versions = ">=3.8" files = [ - {file = "sphinx-autodoc2-0.4.2.tar.gz", hash = "sha256:06da226a25a4339e173b34bb0e590e0ba9b4570b414796140aee1939d09acb3a"}, - {file = "sphinx_autodoc2-0.4.2-py3-none-any.whl", hash = "sha256:00835ba8c980b9c510ea794c3e2060e5a254a74c6c22badc9bfd3642dc1034b4"}, + {file = "sphinx_autodoc2-0.5.0-py3-none-any.whl", hash = "sha256:e867013b1512f9d6d7e6f6799f8b537d6884462acd118ef361f3f619a60b5c9e"}, + {file = "sphinx_autodoc2-0.5.0.tar.gz", hash = "sha256:7d76044aa81d6af74447080182b6868c7eb066874edc835e8ddf810735b6565a"}, ] [package.dependencies] -astroid = ">=2.7" +astroid = ">=2.7,<4" tomli = {version = "*", markers = "python_version < \"3.11\""} typing-extensions = "*" @@ -2709,7 +2709,7 @@ typing-extensions = "*" cli = ["typer[all]"] docs = ["furo", "myst-parser", "sphinx (>=4.0.0)"] sphinx = ["sphinx (>=4.0.0)"] -testing = ["pytest", "pytest-cov", "pytest-regressions", "sphinx (>=4.0.0)"] +testing = ["pytest", "pytest-cov", "pytest-regressions", "sphinx (>=4.0.0,<7)"] [[package]] name = "sphinx-basic-ng" @@ -3432,4 +3432,4 @@ user-search = ["pyicu"] [metadata] lock-version = "2.0" python-versions = "^3.8.0" -content-hash = "2924e80a14b32b430e70bafbd2b00893a365d9c3836c026296f4af0b9579e604" +content-hash = "57716a9580b3493c3d2038492a6d4c36d1d16a79c5a0880b6eadcaf681503d3a" diff --git a/pyproject.toml b/pyproject.toml index b0ccf82e1087..eeede837d093 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -370,7 +370,7 @@ optional = true [tool.poetry.group.dev-docs.dependencies] sphinx = {version = "^6.1", python = "^3.8"} -sphinx-autodoc2 = {version = "^0.4.2", python = "^3.8"} +sphinx-autodoc2 = {version = ">=0.4.2,<0.6.0", python = "^3.8"} myst-parser = {version = "^1.0.0", python = "^3.8"} furo = ">=2022.12.7,<2024.0.0" From c915b918409a4be9ff27382161fd74694b872dfb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Dec 2023 11:30:15 +0000 Subject: [PATCH 14/29] Bump cryptography from 41.0.6 to 41.0.7 (#16721) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 48 ++++++++++++++++++++++++------------------------ 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/poetry.lock b/poetry.lock index c2b8f8f43766..c8636480ed48 100644 --- a/poetry.lock +++ b/poetry.lock @@ -454,34 +454,34 @@ files = [ [[package]] name = "cryptography" -version = "41.0.6" +version = "41.0.7" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." optional = false python-versions = ">=3.7" files = [ - {file = "cryptography-41.0.6-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:0f27acb55a4e77b9be8d550d762b0513ef3fc658cd3eb15110ebbcbd626db12c"}, - {file = "cryptography-41.0.6-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:ae236bb8760c1e55b7a39b6d4d32d2279bc6c7c8500b7d5a13b6fb9fc97be35b"}, - {file = "cryptography-41.0.6-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:afda76d84b053923c27ede5edc1ed7d53e3c9f475ebaf63c68e69f1403c405a8"}, - {file = "cryptography-41.0.6-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:da46e2b5df770070412c46f87bac0849b8d685c5f2679771de277a422c7d0b86"}, - {file = "cryptography-41.0.6-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:ff369dd19e8fe0528b02e8df9f2aeb2479f89b1270d90f96a63500afe9af5cae"}, - {file = "cryptography-41.0.6-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:b648fe2a45e426aaee684ddca2632f62ec4613ef362f4d681a9a6283d10e079d"}, - {file = "cryptography-41.0.6-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:5daeb18e7886a358064a68dbcaf441c036cbdb7da52ae744e7b9207b04d3908c"}, - {file = "cryptography-41.0.6-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:068bc551698c234742c40049e46840843f3d98ad7ce265fd2bd4ec0d11306596"}, - {file = "cryptography-41.0.6-cp37-abi3-win32.whl", hash = "sha256:2132d5865eea673fe6712c2ed5fb4fa49dba10768bb4cc798345748380ee3660"}, - {file = "cryptography-41.0.6-cp37-abi3-win_amd64.whl", hash = "sha256:48783b7e2bef51224020efb61b42704207dde583d7e371ef8fc2a5fb6c0aabc7"}, - {file = "cryptography-41.0.6-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:8efb2af8d4ba9dbc9c9dd8f04d19a7abb5b49eab1f3694e7b5a16a5fc2856f5c"}, - {file = "cryptography-41.0.6-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:c5a550dc7a3b50b116323e3d376241829fd326ac47bc195e04eb33a8170902a9"}, - {file = "cryptography-41.0.6-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:85abd057699b98fce40b41737afb234fef05c67e116f6f3650782c10862c43da"}, - {file = "cryptography-41.0.6-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:f39812f70fc5c71a15aa3c97b2bbe213c3f2a460b79bd21c40d033bb34a9bf36"}, - {file = "cryptography-41.0.6-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:742ae5e9a2310e9dade7932f9576606836ed174da3c7d26bc3d3ab4bd49b9f65"}, - {file = "cryptography-41.0.6-pp38-pypy38_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:35f3f288e83c3f6f10752467c48919a7a94b7d88cc00b0668372a0d2ad4f8ead"}, - {file = "cryptography-41.0.6-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:4d03186af98b1c01a4eda396b137f29e4e3fb0173e30f885e27acec8823c1b09"}, - {file = "cryptography-41.0.6-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:b27a7fd4229abef715e064269d98a7e2909ebf92eb6912a9603c7e14c181928c"}, - {file = "cryptography-41.0.6-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:398ae1fc711b5eb78e977daa3cbf47cec20f2c08c5da129b7a296055fbb22aed"}, - {file = "cryptography-41.0.6-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:7e00fb556bda398b99b0da289ce7053639d33b572847181d6483ad89835115f6"}, - {file = "cryptography-41.0.6-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:60e746b11b937911dc70d164060d28d273e31853bb359e2b2033c9e93e6f3c43"}, - {file = "cryptography-41.0.6-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:3288acccef021e3c3c10d58933f44e8602cf04dba96d9796d70d537bb2f4bbc4"}, - {file = "cryptography-41.0.6.tar.gz", hash = "sha256:422e3e31d63743855e43e5a6fcc8b4acab860f560f9321b0ee6269cc7ed70cc3"}, + {file = "cryptography-41.0.7-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:3c78451b78313fa81607fa1b3f1ae0a5ddd8014c38a02d9db0616133987b9cdf"}, + {file = "cryptography-41.0.7-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:928258ba5d6f8ae644e764d0f996d61a8777559f72dfeb2eea7e2fe0ad6e782d"}, + {file = "cryptography-41.0.7-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5a1b41bc97f1ad230a41657d9155113c7521953869ae57ac39ac7f1bb471469a"}, + {file = "cryptography-41.0.7-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:841df4caa01008bad253bce2a6f7b47f86dc9f08df4b433c404def869f590a15"}, + {file = "cryptography-41.0.7-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:5429ec739a29df2e29e15d082f1d9ad683701f0ec7709ca479b3ff2708dae65a"}, + {file = "cryptography-41.0.7-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:43f2552a2378b44869fe8827aa19e69512e3245a219104438692385b0ee119d1"}, + {file = "cryptography-41.0.7-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:af03b32695b24d85a75d40e1ba39ffe7db7ffcb099fe507b39fd41a565f1b157"}, + {file = "cryptography-41.0.7-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:49f0805fc0b2ac8d4882dd52f4a3b935b210935d500b6b805f321addc8177406"}, + {file = "cryptography-41.0.7-cp37-abi3-win32.whl", hash = "sha256:f983596065a18a2183e7f79ab3fd4c475205b839e02cbc0efbbf9666c4b3083d"}, + {file = "cryptography-41.0.7-cp37-abi3-win_amd64.whl", hash = "sha256:90452ba79b8788fa380dfb587cca692976ef4e757b194b093d845e8d99f612f2"}, + {file = "cryptography-41.0.7-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:079b85658ea2f59c4f43b70f8119a52414cdb7be34da5d019a77bf96d473b960"}, + {file = "cryptography-41.0.7-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:b640981bf64a3e978a56167594a0e97db71c89a479da8e175d8bb5be5178c003"}, + {file = "cryptography-41.0.7-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:e3114da6d7f95d2dee7d3f4eec16dacff819740bbab931aff8648cb13c5ff5e7"}, + {file = "cryptography-41.0.7-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:d5ec85080cce7b0513cfd233914eb8b7bbd0633f1d1703aa28d1dd5a72f678ec"}, + {file = "cryptography-41.0.7-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:7a698cb1dac82c35fcf8fe3417a3aaba97de16a01ac914b89a0889d364d2f6be"}, + {file = "cryptography-41.0.7-pp38-pypy38_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:37a138589b12069efb424220bf78eac59ca68b95696fc622b6ccc1c0a197204a"}, + {file = "cryptography-41.0.7-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:68a2dec79deebc5d26d617bfdf6e8aab065a4f34934b22d3b5010df3ba36612c"}, + {file = "cryptography-41.0.7-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:09616eeaef406f99046553b8a40fbf8b1e70795a91885ba4c96a70793de5504a"}, + {file = "cryptography-41.0.7-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:48a0476626da912a44cc078f9893f292f0b3e4c739caf289268168d8f4702a39"}, + {file = "cryptography-41.0.7-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:c7f3201ec47d5207841402594f1d7950879ef890c0c495052fa62f58283fde1a"}, + {file = "cryptography-41.0.7-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:c5ca78485a255e03c32b513f8c2bc39fedb7f5c5f8535545bdc223a03b24f248"}, + {file = "cryptography-41.0.7-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:d6c391c021ab1f7a82da5d8d0b3cee2f4b2c455ec86c8aebbc84837a631ff309"}, + {file = "cryptography-41.0.7.tar.gz", hash = "sha256:13f93ce9bea8016c253b34afc6bd6a75993e5c40672ed5405a9c832f0d4a00bc"}, ] [package.dependencies] From 2686a05766d666a40f5b8658af17246b46225899 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Dec 2023 11:34:53 +0000 Subject: [PATCH 15/29] Bump idna from 3.4 to 3.6 (#16720) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index c8636480ed48..44a1dae3ac37 100644 --- a/poetry.lock +++ b/poetry.lock @@ -712,13 +712,13 @@ idna = ">=2.5" [[package]] name = "idna" -version = "3.4" +version = "3.6" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.5" files = [ - {file = "idna-3.4-py3-none-any.whl", hash = "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"}, - {file = "idna-3.4.tar.gz", hash = "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4"}, + {file = "idna-3.6-py3-none-any.whl", hash = "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f"}, + {file = "idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca"}, ] [[package]] From d6e194b2bc1b71c04d55617359b43f4dde5bc593 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 4 Dec 2023 04:36:12 -0700 Subject: [PATCH 16/29] Implement MSC4069: Inhibit profile propagation (#16636) MSC: https://github.com/matrix-org/matrix-spec-proposals/pull/4069 --- changelog.d/16636.feature | 1 + synapse/config/experimental.py | 4 + synapse/handlers/profile.py | 10 +- synapse/rest/client/profile.py | 31 +++++- synapse/rest/client/versions.py | 2 + tests/rest/client/test_profile.py | 160 ++++++++++++++++++++++++++++++ 6 files changed, 203 insertions(+), 5 deletions(-) create mode 100644 changelog.d/16636.feature diff --git a/changelog.d/16636.feature b/changelog.d/16636.feature new file mode 100644 index 000000000000..a363eaafaf13 --- /dev/null +++ b/changelog.d/16636.feature @@ -0,0 +1 @@ +Support MSC4069: Inhibit profile propagation. \ No newline at end of file diff --git a/synapse/config/experimental.py b/synapse/config/experimental.py index 9f830e709410..6b9febe5a737 100644 --- a/synapse/config/experimental.py +++ b/synapse/config/experimental.py @@ -419,3 +419,7 @@ def read_config(self, config: JsonDict, **kwargs: Any) -> None: self.msc4028_push_encrypted_events = experimental.get( "msc4028_push_encrypted_events", False ) + + self.msc4069_profile_inhibit_propagation = experimental.get( + "msc4069_profile_inhibit_propagation", False + ) diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py index 1027fbfd2839..e043fd53228b 100644 --- a/synapse/handlers/profile.py +++ b/synapse/handlers/profile.py @@ -129,6 +129,7 @@ async def set_displayname( new_displayname: str, by_admin: bool = False, deactivation: bool = False, + propagate: bool = True, ) -> None: """Set the displayname of a user @@ -138,6 +139,7 @@ async def set_displayname( new_displayname: The displayname to give this user. by_admin: Whether this change was made by an administrator. deactivation: Whether this change was made while deactivating the user. + propagate: Whether this change also applies to the user's membership events. """ if not self.hs.is_mine(target_user): raise SynapseError(400, "User is not hosted on this homeserver") @@ -188,7 +190,8 @@ async def set_displayname( target_user.to_string(), profile, by_admin, deactivation ) - await self._update_join_states(requester, target_user) + if propagate: + await self._update_join_states(requester, target_user) async def get_avatar_url(self, target_user: UserID) -> Optional[str]: if self.hs.is_mine(target_user): @@ -221,6 +224,7 @@ async def set_avatar_url( new_avatar_url: str, by_admin: bool = False, deactivation: bool = False, + propagate: bool = True, ) -> None: """Set a new avatar URL for a user. @@ -230,6 +234,7 @@ async def set_avatar_url( new_avatar_url: The avatar URL to give this user. by_admin: Whether this change was made by an administrator. deactivation: Whether this change was made while deactivating the user. + propagate: Whether this change also applies to the user's membership events. """ if not self.hs.is_mine(target_user): raise SynapseError(400, "User is not hosted on this homeserver") @@ -278,7 +283,8 @@ async def set_avatar_url( target_user.to_string(), profile, by_admin, deactivation ) - await self._update_join_states(requester, target_user) + if propagate: + await self._update_join_states(requester, target_user) @cached() async def check_avatar_size_and_mime_type(self, mxc: str) -> bool: diff --git a/synapse/rest/client/profile.py b/synapse/rest/client/profile.py index 493e1acea06e..12d3da1ced02 100644 --- a/synapse/rest/client/profile.py +++ b/synapse/rest/client/profile.py @@ -13,12 +13,17 @@ # limitations under the License. """ This module contains REST servlets to do with profile: /profile/ """ + from http import HTTPStatus from typing import TYPE_CHECKING, Tuple from synapse.api.errors import Codes, SynapseError from synapse.http.server import HttpServer -from synapse.http.servlet import RestServlet, parse_json_object_from_request +from synapse.http.servlet import ( + RestServlet, + parse_boolean, + parse_json_object_from_request, +) from synapse.http.site import SynapseRequest from synapse.rest.client._base import client_patterns from synapse.types import JsonDict, UserID @@ -27,6 +32,20 @@ from synapse.server import HomeServer +def _read_propagate(hs: "HomeServer", request: SynapseRequest) -> bool: + # This will always be set by the time Twisted calls us. + assert request.args is not None + + propagate = True + if hs.config.experimental.msc4069_profile_inhibit_propagation: + do_propagate = request.args.get(b"org.matrix.msc4069.propagate") + if do_propagate is not None: + propagate = parse_boolean( + request, "org.matrix.msc4069.propagate", default=False + ) + return propagate + + class ProfileDisplaynameRestServlet(RestServlet): PATTERNS = client_patterns("/profile/(?P[^/]*)/displayname", v1=True) CATEGORY = "Event sending requests" @@ -80,7 +99,11 @@ async def on_PUT( errcode=Codes.BAD_JSON, ) - await self.profile_handler.set_displayname(user, requester, new_name, is_admin) + propagate = _read_propagate(self.hs, request) + + await self.profile_handler.set_displayname( + user, requester, new_name, is_admin, propagate=propagate + ) return 200, {} @@ -135,8 +158,10 @@ async def on_PUT( 400, "Missing key 'avatar_url'", errcode=Codes.MISSING_PARAM ) + propagate = _read_propagate(self.hs, request) + await self.profile_handler.set_avatar_url( - user, requester, new_avatar_url, is_admin + user, requester, new_avatar_url, is_admin, propagate=propagate ) return 200, {} diff --git a/synapse/rest/client/versions.py b/synapse/rest/client/versions.py index f4d19e04706d..54c01bb739fc 100644 --- a/synapse/rest/client/versions.py +++ b/synapse/rest/client/versions.py @@ -129,6 +129,8 @@ def on_GET(self, request: Request) -> Tuple[int, JsonDict]: "org.matrix.msc3981": self.config.experimental.msc3981_recurse_relations, # Adds support for deleting account data. "org.matrix.msc3391": self.config.experimental.msc3391_enabled, + # Allows clients to inhibit profile update propagation. + "org.matrix.msc4069": self.config.experimental.msc4069_profile_inhibit_propagation, }, }, ) diff --git a/tests/rest/client/test_profile.py b/tests/rest/client/test_profile.py index 8f923fd40f89..eb0fa00bb342 100644 --- a/tests/rest/client/test_profile.py +++ b/tests/rest/client/test_profile.py @@ -312,6 +312,166 @@ def test_avatar_allowed_mime_type_per_room(self) -> None: ) self.assertEqual(channel.code, 200, channel.result) + @unittest.override_config( + {"experimental_features": {"msc4069_profile_inhibit_propagation": True}} + ) + def test_msc4069_inhibit_propagation(self) -> None: + """Tests to ensure profile update propagation can be inhibited.""" + for prop in ["avatar_url", "displayname"]: + room_id = self.helper.create_room_as(tok=self.owner_tok) + + channel = self.make_request( + "PUT", + f"/rooms/{room_id}/state/m.room.member/{self.owner}", + content={"membership": "join", prop: "mxc://my.server/existing"}, + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + + channel = self.make_request( + "PUT", + f"/profile/{self.owner}/{prop}?org.matrix.msc4069.propagate=false", + content={prop: "http://my.server/pic.gif"}, + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + + res = ( + self._get_avatar_url() + if prop == "avatar_url" + else self._get_displayname() + ) + self.assertEqual(res, "http://my.server/pic.gif") + + channel = self.make_request( + "GET", + f"/rooms/{room_id}/state/m.room.member/{self.owner}", + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + self.assertEqual(channel.json_body.get(prop), "mxc://my.server/existing") + + def test_msc4069_inhibit_propagation_disabled(self) -> None: + """Tests to ensure profile update propagation inhibit flags are ignored when the + experimental flag is not enabled. + """ + for prop in ["avatar_url", "displayname"]: + room_id = self.helper.create_room_as(tok=self.owner_tok) + + channel = self.make_request( + "PUT", + f"/rooms/{room_id}/state/m.room.member/{self.owner}", + content={"membership": "join", prop: "mxc://my.server/existing"}, + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + + channel = self.make_request( + "PUT", + f"/profile/{self.owner}/{prop}?org.matrix.msc4069.propagate=false", + content={prop: "http://my.server/pic.gif"}, + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + + res = ( + self._get_avatar_url() + if prop == "avatar_url" + else self._get_displayname() + ) + self.assertEqual(res, "http://my.server/pic.gif") + + channel = self.make_request( + "GET", + f"/rooms/{room_id}/state/m.room.member/{self.owner}", + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + + # The ?propagate=false should be ignored by the server because the config flag + # isn't enabled. + self.assertEqual(channel.json_body.get(prop), "http://my.server/pic.gif") + + def test_msc4069_inhibit_propagation_default(self) -> None: + """Tests to ensure profile update propagation happens by default.""" + for prop in ["avatar_url", "displayname"]: + room_id = self.helper.create_room_as(tok=self.owner_tok) + + channel = self.make_request( + "PUT", + f"/rooms/{room_id}/state/m.room.member/{self.owner}", + content={"membership": "join", prop: "mxc://my.server/existing"}, + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + + channel = self.make_request( + "PUT", + f"/profile/{self.owner}/{prop}", + content={prop: "http://my.server/pic.gif"}, + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + + res = ( + self._get_avatar_url() + if prop == "avatar_url" + else self._get_displayname() + ) + self.assertEqual(res, "http://my.server/pic.gif") + + channel = self.make_request( + "GET", + f"/rooms/{room_id}/state/m.room.member/{self.owner}", + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + + # The ?propagate=false should be ignored by the server because the config flag + # isn't enabled. + self.assertEqual(channel.json_body.get(prop), "http://my.server/pic.gif") + + @unittest.override_config( + {"experimental_features": {"msc4069_profile_inhibit_propagation": True}} + ) + def test_msc4069_inhibit_propagation_like_default(self) -> None: + """Tests to ensure clients can request explicit profile propagation.""" + for prop in ["avatar_url", "displayname"]: + room_id = self.helper.create_room_as(tok=self.owner_tok) + + channel = self.make_request( + "PUT", + f"/rooms/{room_id}/state/m.room.member/{self.owner}", + content={"membership": "join", prop: "mxc://my.server/existing"}, + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + + channel = self.make_request( + "PUT", + f"/profile/{self.owner}/{prop}?org.matrix.msc4069.propagate=true", + content={prop: "http://my.server/pic.gif"}, + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + + res = ( + self._get_avatar_url() + if prop == "avatar_url" + else self._get_displayname() + ) + self.assertEqual(res, "http://my.server/pic.gif") + + channel = self.make_request( + "GET", + f"/rooms/{room_id}/state/m.room.member/{self.owner}", + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + + # The client requested ?propagate=true, so it should have happened. + self.assertEqual(channel.json_body.get(prop), "http://my.server/pic.gif") + def _setup_local_files(self, names_and_props: Dict[str, Dict[str, Any]]) -> None: """Stores metadata about files in the database. From 506f5c75531eb88a8c39bbb523a9cedb32b18c59 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Dec 2023 12:02:54 +0000 Subject: [PATCH 17/29] Bump matrix-org/netlify-pr-preview from 2 to 3 (#16719) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/docs-pr-netlify.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docs-pr-netlify.yaml b/.github/workflows/docs-pr-netlify.yaml index b443cd87d1fb..8b2032230881 100644 --- a/.github/workflows/docs-pr-netlify.yaml +++ b/.github/workflows/docs-pr-netlify.yaml @@ -22,7 +22,7 @@ jobs: path: book - name: 📤 Deploy to Netlify - uses: matrix-org/netlify-pr-preview@v2 + uses: matrix-org/netlify-pr-preview@v3 with: path: book owner: ${{ github.event.workflow_run.head_repository.owner.login }} From 9e7f80037d08619cdd193831540d34e9ed5aacd7 Mon Sep 17 00:00:00 2001 From: Mathieu Velten Date: Mon, 4 Dec 2023 13:31:42 +0100 Subject: [PATCH 18/29] Server notices: add an autojoin setting for the notices room (#16699) Co-authored-by: Patrick Cloke --- changelog.d/16699.feature | 1 + docs/server_notices.md | 3 +++ .../configuration/config_documentation.md | 3 +++ synapse/config/server_notices.py | 2 ++ .../server_notices/server_notices_manager.py | 15 ++++++++++- tests/rest/admin/test_server_notice.py | 27 +++++++++++++++++++ 6 files changed, 50 insertions(+), 1 deletion(-) create mode 100644 changelog.d/16699.feature diff --git a/changelog.d/16699.feature b/changelog.d/16699.feature new file mode 100644 index 000000000000..7ede50f3264c --- /dev/null +++ b/changelog.d/16699.feature @@ -0,0 +1 @@ +Add an autojoin setting for the notices room so users get joined directly instead of receiving an invite. diff --git a/docs/server_notices.md b/docs/server_notices.md index 339d10a0ab3f..aae25d23b82e 100644 --- a/docs/server_notices.md +++ b/docs/server_notices.md @@ -46,6 +46,7 @@ server_notices: system_mxid_display_name: "Server Notices" system_mxid_avatar_url: "mxc://server.com/oumMVlgDnLYFaPVkExemNVVZ" room_name: "Server Notices" + auto_join: true ``` The only compulsory setting is `system_mxid_localpart`, which defines the user @@ -55,6 +56,8 @@ room which will be created. `system_mxid_display_name` and `system_mxid_avatar_url` can be used to set the displayname and avatar of the Server Notices user. +`auto_join` will autojoin users to the notices room instead of sending an invite. + ## Sending notices To send server notices to users you can use the diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index 7c4e742cd5d5..812a7d429b76 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -3815,6 +3815,8 @@ Sub-options for this setting include: * `system_mxid_display_name`: set the display name of the "notices" user * `system_mxid_avatar_url`: set the avatar for the "notices" user * `room_name`: set the room name of the server notices room +* `auto_join`: boolean. If true, the user will be automatically joined to the room instead of being invited. + Defaults to false. _Added in Synapse 1.98.0._ Example configuration: ```yaml @@ -3823,6 +3825,7 @@ server_notices: system_mxid_display_name: "Server Notices" system_mxid_avatar_url: "mxc://server.com/oumMVlgDnLYFaPVkExemNVVZ" room_name: "Server Notices" + auto_join: true ``` --- ### `enable_room_list_search` diff --git a/synapse/config/server_notices.py b/synapse/config/server_notices.py index ce041abe9bb3..a8badba0f872 100644 --- a/synapse/config/server_notices.py +++ b/synapse/config/server_notices.py @@ -48,6 +48,7 @@ def __init__(self, *args: Any): self.server_notices_mxid_display_name: Optional[str] = None self.server_notices_mxid_avatar_url: Optional[str] = None self.server_notices_room_name: Optional[str] = None + self.server_notices_auto_join: bool = False def read_config(self, config: JsonDict, **kwargs: Any) -> None: c = config.get("server_notices") @@ -62,3 +63,4 @@ def read_config(self, config: JsonDict, **kwargs: Any) -> None: self.server_notices_mxid_avatar_url = c.get("system_mxid_avatar_url", None) # todo: i18n self.server_notices_room_name = c.get("room_name", "Server Notices") + self.server_notices_auto_join = c.get("auto_join", False) diff --git a/synapse/server_notices/server_notices_manager.py b/synapse/server_notices/server_notices_manager.py index 44b999677a1e..2353b5d47fb6 100644 --- a/synapse/server_notices/server_notices_manager.py +++ b/synapse/server_notices/server_notices_manager.py @@ -224,14 +224,27 @@ async def maybe_invite_user_to_room(self, user_id: str, room_id: str) -> None: if room.room_id == room_id: return + user_id_obj = UserID.from_string(user_id) await self._room_member_handler.update_membership( requester=requester, - target=UserID.from_string(user_id), + target=user_id_obj, room_id=room_id, action="invite", ratelimit=False, ) + if self._config.servernotices.server_notices_auto_join: + user_requester = create_requester( + user_id, authenticated_entity=self._server_name + ) + await self._room_member_handler.update_membership( + requester=user_requester, + target=user_id_obj, + room_id=room_id, + action="join", + ratelimit=False, + ) + async def _update_notice_user_profile_if_changed( self, requester: Requester, diff --git a/tests/rest/admin/test_server_notice.py b/tests/rest/admin/test_server_notice.py index dfd14f5751bf..2398bc503a4d 100644 --- a/tests/rest/admin/test_server_notice.py +++ b/tests/rest/admin/test_server_notice.py @@ -477,6 +477,33 @@ def test_send_server_notice_delete_room(self) -> None: # second room has new ID self.assertNotEqual(first_room_id, second_room_id) + @override_config( + {"server_notices": {"system_mxid_localpart": "notices", "auto_join": True}} + ) + def test_auto_join(self) -> None: + """ + Tests that the user get automatically joined to the notice room + when `auto_join` setting is used. + """ + # user has no room memberships + self._check_invite_and_join_status(self.other_user, 0, 0) + + # send server notice + server_notice_request_content = { + "user_id": self.other_user, + "content": {"msgtype": "m.text", "body": "test msg one"}, + } + + self.make_request( + "POST", + self.url, + access_token=self.admin_user_tok, + content=server_notice_request_content, + ) + + # user has joined the room + self._check_invite_and_join_status(self.other_user, 0, 1) + @override_config({"server_notices": {"system_mxid_localpart": "notices"}}) def test_update_notice_user_name_when_changed(self) -> None: """ From 15c46cf86a6e04cbea92067145739ed11b9f655e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Dec 2023 12:32:06 +0000 Subject: [PATCH 19/29] Bump phonenumbers from 8.13.23 to 8.13.26 (#16722) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index 44a1dae3ac37..89387db20aa0 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1611,13 +1611,13 @@ files = [ [[package]] name = "phonenumbers" -version = "8.13.23" +version = "8.13.26" description = "Python version of Google's common library for parsing, formatting, storing and validating international phone numbers." optional = false python-versions = "*" files = [ - {file = "phonenumbers-8.13.23-py2.py3-none-any.whl", hash = "sha256:34d6cb279dd4a64714e324c71350f96e5bda3237be28d11b4c555c44701544cd"}, - {file = "phonenumbers-8.13.23.tar.gz", hash = "sha256:869e44fcaaf276eca6b953a401e2b27d57461f3a18a66cf5f13377e7bb0e228c"}, + {file = "phonenumbers-8.13.26-py2.py3-none-any.whl", hash = "sha256:b2308c9c5750b8f10dd30d94547afd66bce60ac5e93aff227f95740557f32752"}, + {file = "phonenumbers-8.13.26.tar.gz", hash = "sha256:937d70aeceb317f5831dfec28de855a60260ef4a9d551964bec8e7a7d0cf81cd"}, ] [[package]] From 0aa4d3b6f7193d9721bd484dae0242ccd0cc10b8 Mon Sep 17 00:00:00 2001 From: villepeh <100730729+villepeh@users.noreply.github.com> Date: Mon, 4 Dec 2023 14:38:46 +0200 Subject: [PATCH 20/29] Switch UNIX socket paths to /run, and add a UNIX socket example for HAProxy (#16700) --- changelog.d/16700.doc | 1 + docs/reverse_proxy.md | 6 +++++- docs/usage/configuration/config_documentation.md | 14 +++++++------- 3 files changed, 13 insertions(+), 8 deletions(-) create mode 100644 changelog.d/16700.doc diff --git a/changelog.d/16700.doc b/changelog.d/16700.doc new file mode 100644 index 000000000000..0d55ad338a62 --- /dev/null +++ b/changelog.d/16700.doc @@ -0,0 +1 @@ +Switch the example UNIX socket paths to /run. Add HAProxy example configuration for UNIX sockets. diff --git a/docs/reverse_proxy.md b/docs/reverse_proxy.md index fe9519b4b624..20854035d11c 100644 --- a/docs/reverse_proxy.md +++ b/docs/reverse_proxy.md @@ -181,7 +181,11 @@ frontend matrix-federation backend matrix server matrix 127.0.0.1:8008 ``` - +Example configuration, if using a UNIX socket. The configuration lines regarding the frontends do not need to be modified. +``` +backend matrix + server matrix unix@/run/synapse/main_public.sock +``` [Delegation](delegate.md) example: ``` diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index 812a7d429b76..e6fcffb5706b 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -566,7 +566,7 @@ listeners: # Note that x_forwarded will default to true, when using a UNIX socket. Please see # https://matrix-org.github.io/synapse/latest/reverse_proxy.html. # - - path: /var/run/synapse/main_public.sock + - path: /run/synapse/main_public.sock type: http resources: - names: [client, federation] @@ -4218,9 +4218,9 @@ Example configuration(#2, for UNIX sockets): ```yaml instance_map: main: - path: /var/run/synapse/main_replication.sock + path: /run/synapse/main_replication.sock worker1: - path: /var/run/synapse/worker1_replication.sock + path: /run/synapse/worker1_replication.sock ``` --- ### `stream_writers` @@ -4406,13 +4406,13 @@ Example configuration(#2, using UNIX sockets with a `replication` listener): ```yaml worker_listeners: - type: http - path: /var/run/synapse/worker_public.sock + path: /run/synapse/worker_replication.sock resources: - - names: [client, federation] + - names: [replication] - type: http - path: /var/run/synapse/worker_replication.sock + path: /run/synapse/worker_public.sock resources: - - names: [replication] + - names: [client, federation] ``` --- ### `worker_manhole` From 51e4e35653f98c3f61222fbdbdb1dcb8864f7fca Mon Sep 17 00:00:00 2001 From: reivilibre Date: Mon, 4 Dec 2023 14:57:28 +0000 Subject: [PATCH 21/29] Add a Postgres `REPLICA IDENTITY` to tables that do not have an implicit one. This should allow use of Postgres logical replication. (take 2, now with no added deadlocks!) (#16658) * Add `ALTER TABLE ... REPLICA IDENTITY ...` for individual tables We can't combine them into one file as it makes it likely to hit a deadlock if Synapse is running, as it only takes one other transaction to access two tables in a different order to the schema delta. * Add notes * Newsfile Signed-off-by: Olivier Wilkinson (reivilibre) * Re-introduce REPLICA IDENTITY test --------- Signed-off-by: Olivier Wilkinson (reivilibre) --- changelog.d/16658.misc | 1 + ...entity_applied_module_schemas.sql.postgres | 1 + ...dentity_applied_schema_deltas.sql.postgres | 1 + ...a_identity_background_updates.sql.postgres | 1 + ...dentity_schema_compat_version.sql.postgres | 1 + ...plica_identity_schema_version.sql.postgres | 1 + .../delta/83/10_replica_identity_README.md | 13 +++ ...replica_identity_account_data.sql.postgres | 1 + ...ity_application_services_txns.sql.postgres | 1 + ...identity_appservice_room_list.sql.postgres | 1 + ...ty_appservice_stream_position.sql.postgres | 1 + ...replica_identity_batch_events.sql.postgres | 1 + ...eplica_identity_blocked_rooms.sql.postgres | 1 + ...validation_stream_by_instance.sql.postgres | 1 + ...ty_current_state_delta_stream.sql.postgres | 1 + ...identity_current_state_events.sql.postgres | 1 + ...lica_identity_deleted_pushers.sql.postgres | 1 + ...dentity_device_auth_providers.sql.postgres | 1 + ...ntity_device_federation_inbox.sql.postgres | 1 + ...tity_device_federation_outbox.sql.postgres | 1 + ...replica_identity_device_inbox.sql.postgres | 1 + ...ges_converted_stream_position.sql.postgres | 1 + ..._device_lists_changes_in_room.sql.postgres | 1 + ...e_lists_outbound_last_success.sql.postgres | 1 + ...y_device_lists_outbound_pokes.sql.postgres | 1 + ...ity_device_lists_remote_cache.sql.postgres | 1 + ...vice_lists_remote_extremeties.sql.postgres | 1 + ...ty_device_lists_remote_resync.sql.postgres | 1 + ..._identity_device_lists_stream.sql.postgres | 1 + .../10_replica_identity_devices.sql.postgres | 1 + ...entity_e2e_cross_signing_keys.sql.postgres | 1 + ..._e2e_cross_signing_signatures.sql.postgres | 1 + ...identity_e2e_device_keys_json.sql.postgres | 1 + ...entity_e2e_fallback_keys_json.sql.postgres | 1 + ...entity_e2e_one_time_keys_json.sql.postgres | 1 + ...eplica_identity_e2e_room_keys.sql.postgres | 1 + ...entity_e2e_room_keys_versions.sql.postgres | 1 + ...replica_identity_erased_users.sql.postgres | 1 + ...0_replica_identity_event_auth.sql.postgres | 1 + ...entity_event_auth_chain_links.sql.postgres | 1 + ...ty_event_backward_extremities.sql.postgres | 1 + ..._replica_identity_event_edges.sql.postgres | 1 + ...ity_event_forward_extremities.sql.postgres | 1 + ...0_replica_identity_event_json.sql.postgres | 1 + ...a_identity_event_push_actions.sql.postgres | 1 + ...ty_event_push_actions_staging.sql.postgres | 1 + ...a_identity_event_push_summary.sql.postgres | 1 + ...ummary_last_receipt_stream_id.sql.postgres | 1 + ..._push_summary_stream_ordering.sql.postgres | 1 + ...lica_identity_event_relations.sql.postgres | 1 + ...replica_identity_event_search.sql.postgres | 1 + ...dentity_event_to_state_groups.sql.postgres | 1 + ...replica_identity_event_txn_id.sql.postgres | 1 + ...entity_event_txn_id_device_id.sql.postgres | 1 + .../10_replica_identity_events.sql.postgres | 1 + ...ration_inbound_events_staging.sql.postgres | 1 + ...ty_federation_stream_position.sql.postgres | 1 + ...eplica_identity_ignored_users.sql.postgres | 1 + ...dentity_insertion_event_edges.sql.postgres | 1 + ...y_insertion_event_extremities.sql.postgres | 1 + ...ica_identity_insertion_events.sql.postgres | 1 + ...tity_local_current_membership.sql.postgres | 1 + ...entity_local_media_repository.sql.postgres | 1 + ...l_media_repository_thumbnails.sql.postgres | 1 + ...al_media_repository_url_cache.sql.postgres | 1 + ...identity_monthly_active_users.sql.postgres | 1 + ...identity_partial_state_events.sql.postgres | 1 + ...y_partial_state_rooms_servers.sql.postgres | 1 + ...lica_identity_presence_stream.sql.postgres | 1 + .../10_replica_identity_profiles.sql.postgres | 1 + ...ca_identity_push_rules_stream.sql.postgres | 1 + ...a_identity_ratelimit_override.sql.postgres | 1 + ...plica_identity_receipts_graph.sql.postgres | 1 + ..._identity_receipts_linearized.sql.postgres | 1 + ...dentity_received_transactions.sql.postgres | 1 + ...0_replica_identity_redactions.sql.postgres | 1 + ..._identity_registration_tokens.sql.postgres | 1 + ...0_replica_identity_rejections.sql.postgres | 1 + ...a_identity_remote_media_cache.sql.postgres | 1 + ...remote_media_cache_thumbnails.sql.postgres | 1 + ...ca_identity_room_account_data.sql.postgres | 1 + ...a_identity_room_alias_servers.sql.postgres | 1 + ...replica_identity_room_aliases.sql.postgres | 1 + ...0_replica_identity_room_depth.sql.postgres | 1 + ...ity_room_forgetter_stream_pos.sql.postgres | 1 + ...ica_identity_room_memberships.sql.postgres | 1 + ...ity_room_stats_earliest_token.sql.postgres | 1 + ...ica_identity_room_stats_state.sql.postgres | 1 + ...10_replica_identity_room_tags.sql.postgres | 1 + ..._identity_room_tags_revisions.sql.postgres | 1 + ...ica_identity_server_keys_json.sql.postgres | 1 + ...dentity_server_signature_keys.sql.postgres | 1 + .../10_replica_identity_sessions.sql.postgres | 1 + ...replica_identity_state_events.sql.postgres | 1 + ...ty_stats_incremental_position.sql.postgres | 1 + ...ity_stream_ordering_to_exterm.sql.postgres | 1 + ...ica_identity_stream_positions.sql.postgres | 1 + .../10_replica_identity_threads.sql.postgres | 1 + ..._threepid_guest_access_tokens.sql.postgres | 1 + ...eplica_identity_timeline_gaps.sql.postgres | 1 + ...ica_identity_ui_auth_sessions.sql.postgres | 1 + ..._ui_auth_sessions_credentials.sql.postgres | 1 + ...identity_ui_auth_sessions_ips.sql.postgres | 1 + ...ca_identity_user_daily_visits.sql.postgres | 1 + ...plica_identity_user_directory.sql.postgres | 1 + ...dentity_user_directory_search.sql.postgres | 1 + ...ity_user_directory_stream_pos.sql.postgres | 1 + ...ca_identity_user_external_ids.sql.postgres | 1 + ...replica_identity_user_filters.sql.postgres | 1 + .../10_replica_identity_user_ips.sql.postgres | 1 + ...dentity_user_signature_stream.sql.postgres | 1 + ...ntity_user_threepid_id_server.sql.postgres | 1 + ...plica_identity_user_threepids.sql.postgres | 1 + .../83/10_replica_identity_users.sql.postgres | 1 + ...dentity_users_in_public_rooms.sql.postgres | 1 + ...ty_users_pending_deactivation.sql.postgres | 1 + ...users_who_share_private_rooms.sql.postgres | 1 + ...replica_identity_worker_locks.sql.postgres | 1 + ...ntity_worker_read_write_locks.sql.postgres | 1 + ..._worker_read_write_locks_mode.sql.postgres | 1 + ...ca_identity_state_group_edges.sql.postgres | 1 + ...a_identity_state_groups_state.sql.postgres | 1 + tests/storage/test_database.py | 85 ++++++++++++++++++- 123 files changed, 218 insertions(+), 1 deletion(-) create mode 100644 changelog.d/16658.misc create mode 100644 synapse/storage/schema/common/delta/83/10_replica_identity_applied_module_schemas.sql.postgres create mode 100644 synapse/storage/schema/common/delta/83/10_replica_identity_applied_schema_deltas.sql.postgres create mode 100644 synapse/storage/schema/common/delta/83/10_replica_identity_background_updates.sql.postgres create mode 100644 synapse/storage/schema/common/delta/83/10_replica_identity_schema_compat_version.sql.postgres create mode 100644 synapse/storage/schema/common/delta/83/10_replica_identity_schema_version.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_README.md create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_account_data.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_application_services_txns.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_appservice_room_list.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_appservice_stream_position.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_batch_events.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_blocked_rooms.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_cache_invalidation_stream_by_instance.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_current_state_delta_stream.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_current_state_events.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_deleted_pushers.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_device_auth_providers.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_device_federation_inbox.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_device_federation_outbox.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_device_inbox.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_device_lists_changes_converted_stream_position.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_device_lists_changes_in_room.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_device_lists_outbound_last_success.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_device_lists_outbound_pokes.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_device_lists_remote_cache.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_device_lists_remote_extremeties.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_device_lists_remote_resync.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_device_lists_stream.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_devices.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_e2e_cross_signing_keys.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_e2e_cross_signing_signatures.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_e2e_device_keys_json.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_e2e_fallback_keys_json.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_e2e_one_time_keys_json.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_e2e_room_keys.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_e2e_room_keys_versions.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_erased_users.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_event_auth.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_event_auth_chain_links.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_event_backward_extremities.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_event_edges.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_event_forward_extremities.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_event_json.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_event_push_actions.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_event_push_actions_staging.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_event_push_summary.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_event_push_summary_last_receipt_stream_id.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_event_push_summary_stream_ordering.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_event_relations.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_event_search.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_event_to_state_groups.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_event_txn_id.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_event_txn_id_device_id.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_events.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_federation_inbound_events_staging.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_federation_stream_position.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_ignored_users.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_insertion_event_edges.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_insertion_event_extremities.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_insertion_events.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_local_current_membership.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_local_media_repository.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_local_media_repository_thumbnails.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_local_media_repository_url_cache.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_monthly_active_users.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_partial_state_events.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_partial_state_rooms_servers.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_presence_stream.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_profiles.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_push_rules_stream.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_ratelimit_override.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_receipts_graph.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_receipts_linearized.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_received_transactions.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_redactions.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_registration_tokens.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_rejections.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_remote_media_cache.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_remote_media_cache_thumbnails.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_room_account_data.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_room_alias_servers.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_room_aliases.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_room_depth.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_room_forgetter_stream_pos.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_room_memberships.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_room_stats_earliest_token.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_room_stats_state.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_room_tags.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_room_tags_revisions.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_server_keys_json.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_server_signature_keys.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_sessions.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_state_events.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_stats_incremental_position.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_stream_ordering_to_exterm.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_stream_positions.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_threads.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_threepid_guest_access_tokens.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_timeline_gaps.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_ui_auth_sessions.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_ui_auth_sessions_credentials.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_ui_auth_sessions_ips.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_user_daily_visits.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_user_directory.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_user_directory_search.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_user_directory_stream_pos.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_user_external_ids.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_user_filters.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_user_ips.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_user_signature_stream.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_user_threepid_id_server.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_user_threepids.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_users.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_users_in_public_rooms.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_users_pending_deactivation.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_users_who_share_private_rooms.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_worker_locks.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_worker_read_write_locks.sql.postgres create mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_worker_read_write_locks_mode.sql.postgres create mode 100644 synapse/storage/schema/state/delta/83/10_replica_identity_state_group_edges.sql.postgres create mode 100644 synapse/storage/schema/state/delta/83/10_replica_identity_state_groups_state.sql.postgres diff --git a/changelog.d/16658.misc b/changelog.d/16658.misc new file mode 100644 index 000000000000..baee042f2f2c --- /dev/null +++ b/changelog.d/16658.misc @@ -0,0 +1 @@ +Add a Postgres `REPLICA IDENTITY` to tables that do not have an implicit one. This should allow use of Postgres logical replication. \ No newline at end of file diff --git a/synapse/storage/schema/common/delta/83/10_replica_identity_applied_module_schemas.sql.postgres b/synapse/storage/schema/common/delta/83/10_replica_identity_applied_module_schemas.sql.postgres new file mode 100644 index 000000000000..57c35b64301a --- /dev/null +++ b/synapse/storage/schema/common/delta/83/10_replica_identity_applied_module_schemas.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE applied_module_schemas REPLICA IDENTITY USING INDEX applied_module_schemas_module_name_file_key; diff --git a/synapse/storage/schema/common/delta/83/10_replica_identity_applied_schema_deltas.sql.postgres b/synapse/storage/schema/common/delta/83/10_replica_identity_applied_schema_deltas.sql.postgres new file mode 100644 index 000000000000..1426f9a9e402 --- /dev/null +++ b/synapse/storage/schema/common/delta/83/10_replica_identity_applied_schema_deltas.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE applied_schema_deltas REPLICA IDENTITY USING INDEX applied_schema_deltas_version_file_key; diff --git a/synapse/storage/schema/common/delta/83/10_replica_identity_background_updates.sql.postgres b/synapse/storage/schema/common/delta/83/10_replica_identity_background_updates.sql.postgres new file mode 100644 index 000000000000..f04937b3db35 --- /dev/null +++ b/synapse/storage/schema/common/delta/83/10_replica_identity_background_updates.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE background_updates REPLICA IDENTITY USING INDEX background_updates_uniqueness; diff --git a/synapse/storage/schema/common/delta/83/10_replica_identity_schema_compat_version.sql.postgres b/synapse/storage/schema/common/delta/83/10_replica_identity_schema_compat_version.sql.postgres new file mode 100644 index 000000000000..e3a6f7ca4ffe --- /dev/null +++ b/synapse/storage/schema/common/delta/83/10_replica_identity_schema_compat_version.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE schema_compat_version REPLICA IDENTITY USING INDEX schema_compat_version_lock_key; diff --git a/synapse/storage/schema/common/delta/83/10_replica_identity_schema_version.sql.postgres b/synapse/storage/schema/common/delta/83/10_replica_identity_schema_version.sql.postgres new file mode 100644 index 000000000000..c10efe05637a --- /dev/null +++ b/synapse/storage/schema/common/delta/83/10_replica_identity_schema_version.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE schema_version REPLICA IDENTITY USING INDEX schema_version_lock_key; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_README.md b/synapse/storage/schema/main/delta/83/10_replica_identity_README.md new file mode 100644 index 000000000000..fd7bdfc7adea --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_README.md @@ -0,0 +1,13 @@ +The `10_replica_identity_xxx.sql.postgres` series of schema deltas adds replica identities for tables that do not have one implicitly as a result of having a primary key. + +This is needed to use logical replication with Synapse (at least without `UPDATE` and `DELETE` statements failing!). + +Where possible, we use an existing `UNIQUE` index on `NOT NULL` columns as the replica identity. Otherwise, we have to fall back to using the full row as a replica identity. + +Unfortunately, by running all the `ALTER TABLE` statements in one schema delta per database, it was too likely to hit a deadlock as it would only take +one other transaction from a running Synapse worker to access the tables out of order and trigger a deadlock. + +By having each statement in its own delta file, each one is run in its own transaction and only needs to take a very brief (instant) lock on the table but no other tables, +so there should be no chance of deadlock. + +Like many schema deltas we already apply to Synapse, it is probably blocked by an ongoing `pg_dump`. diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_account_data.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_account_data.sql.postgres new file mode 100644 index 000000000000..1bf720e80ee2 --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_account_data.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE account_data REPLICA IDENTITY USING INDEX account_data_uniqueness; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_application_services_txns.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_application_services_txns.sql.postgres new file mode 100644 index 000000000000..4ec052ebebb7 --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_application_services_txns.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE application_services_txns REPLICA IDENTITY USING INDEX application_services_txns_as_id_txn_id_key; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_appservice_room_list.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_appservice_room_list.sql.postgres new file mode 100644 index 000000000000..232a628ca2ab --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_appservice_room_list.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE appservice_room_list REPLICA IDENTITY USING INDEX appservice_room_list_idx; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_appservice_stream_position.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_appservice_stream_position.sql.postgres new file mode 100644 index 000000000000..e870a68723e5 --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_appservice_stream_position.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE appservice_stream_position REPLICA IDENTITY USING INDEX appservice_stream_position_lock_key; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_batch_events.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_batch_events.sql.postgres new file mode 100644 index 000000000000..cd4471f4d7c7 --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_batch_events.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE batch_events REPLICA IDENTITY USING INDEX chunk_events_event_id; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_blocked_rooms.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_blocked_rooms.sql.postgres new file mode 100644 index 000000000000..e3fce4045ff4 --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_blocked_rooms.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE blocked_rooms REPLICA IDENTITY USING INDEX blocked_rooms_idx; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_cache_invalidation_stream_by_instance.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_cache_invalidation_stream_by_instance.sql.postgres new file mode 100644 index 000000000000..4fd87faaf62d --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_cache_invalidation_stream_by_instance.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE cache_invalidation_stream_by_instance REPLICA IDENTITY USING INDEX cache_invalidation_stream_by_instance_id; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_current_state_delta_stream.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_current_state_delta_stream.sql.postgres new file mode 100644 index 000000000000..b618286e8a99 --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_current_state_delta_stream.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE current_state_delta_stream REPLICA IDENTITY FULL; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_current_state_events.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_current_state_events.sql.postgres new file mode 100644 index 000000000000..b058a9f06e0d --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_current_state_events.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE current_state_events REPLICA IDENTITY USING INDEX current_state_events_event_id_key; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_deleted_pushers.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_deleted_pushers.sql.postgres new file mode 100644 index 000000000000..0b4fcc5ae158 --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_deleted_pushers.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE deleted_pushers REPLICA IDENTITY FULL; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_device_auth_providers.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_device_auth_providers.sql.postgres new file mode 100644 index 000000000000..8e02438b476a --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_device_auth_providers.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE device_auth_providers REPLICA IDENTITY FULL; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_device_federation_inbox.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_device_federation_inbox.sql.postgres new file mode 100644 index 000000000000..547e485f0071 --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_device_federation_inbox.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE device_federation_inbox REPLICA IDENTITY FULL; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_device_federation_outbox.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_device_federation_outbox.sql.postgres new file mode 100644 index 000000000000..809afe4dc5ff --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_device_federation_outbox.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE device_federation_outbox REPLICA IDENTITY FULL; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_device_inbox.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_device_inbox.sql.postgres new file mode 100644 index 000000000000..07652038a262 --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_device_inbox.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE device_inbox REPLICA IDENTITY FULL; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_device_lists_changes_converted_stream_position.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_device_lists_changes_converted_stream_position.sql.postgres new file mode 100644 index 000000000000..8474b8c84288 --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_device_lists_changes_converted_stream_position.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE device_lists_changes_converted_stream_position REPLICA IDENTITY USING INDEX device_lists_changes_converted_stream_position_lock_key; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_device_lists_changes_in_room.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_device_lists_changes_in_room.sql.postgres new file mode 100644 index 000000000000..712a54bc0f5c --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_device_lists_changes_in_room.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE device_lists_changes_in_room REPLICA IDENTITY USING INDEX device_lists_changes_in_stream_id; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_device_lists_outbound_last_success.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_device_lists_outbound_last_success.sql.postgres new file mode 100644 index 000000000000..4357690e8de3 --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_device_lists_outbound_last_success.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE device_lists_outbound_last_success REPLICA IDENTITY USING INDEX device_lists_outbound_last_success_unique_idx; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_device_lists_outbound_pokes.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_device_lists_outbound_pokes.sql.postgres new file mode 100644 index 000000000000..cc724877cffc --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_device_lists_outbound_pokes.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE device_lists_outbound_pokes REPLICA IDENTITY FULL; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_device_lists_remote_cache.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_device_lists_remote_cache.sql.postgres new file mode 100644 index 000000000000..ea4351c65473 --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_device_lists_remote_cache.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE device_lists_remote_cache REPLICA IDENTITY USING INDEX device_lists_remote_cache_unique_id; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_device_lists_remote_extremeties.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_device_lists_remote_extremeties.sql.postgres new file mode 100644 index 000000000000..9dc3bec6fa57 --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_device_lists_remote_extremeties.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE device_lists_remote_extremeties REPLICA IDENTITY USING INDEX device_lists_remote_extremeties_unique_idx; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_device_lists_remote_resync.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_device_lists_remote_resync.sql.postgres new file mode 100644 index 000000000000..df3cecfb9439 --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_device_lists_remote_resync.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE device_lists_remote_resync REPLICA IDENTITY USING INDEX device_lists_remote_resync_idx; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_device_lists_stream.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_device_lists_stream.sql.postgres new file mode 100644 index 000000000000..30a79226139e --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_device_lists_stream.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE device_lists_stream REPLICA IDENTITY FULL; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_devices.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_devices.sql.postgres new file mode 100644 index 000000000000..57d54a0dc0b5 --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_devices.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE devices REPLICA IDENTITY USING INDEX device_uniqueness; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_e2e_cross_signing_keys.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_e2e_cross_signing_keys.sql.postgres new file mode 100644 index 000000000000..f3f26936313f --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_e2e_cross_signing_keys.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE e2e_cross_signing_keys REPLICA IDENTITY USING INDEX e2e_cross_signing_keys_stream_idx; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_e2e_cross_signing_signatures.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_e2e_cross_signing_signatures.sql.postgres new file mode 100644 index 000000000000..d2347b71538e --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_e2e_cross_signing_signatures.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE e2e_cross_signing_signatures REPLICA IDENTITY FULL; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_e2e_device_keys_json.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_e2e_device_keys_json.sql.postgres new file mode 100644 index 000000000000..66dafdc8e1e7 --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_e2e_device_keys_json.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE e2e_device_keys_json REPLICA IDENTITY USING INDEX e2e_device_keys_json_uniqueness; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_e2e_fallback_keys_json.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_e2e_fallback_keys_json.sql.postgres new file mode 100644 index 000000000000..8bb4dfc78b5d --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_e2e_fallback_keys_json.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE e2e_fallback_keys_json REPLICA IDENTITY USING INDEX e2e_fallback_keys_json_uniqueness; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_e2e_one_time_keys_json.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_e2e_one_time_keys_json.sql.postgres new file mode 100644 index 000000000000..b6609cfdfbc1 --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_e2e_one_time_keys_json.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE e2e_one_time_keys_json REPLICA IDENTITY USING INDEX e2e_one_time_keys_json_uniqueness; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_e2e_room_keys.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_e2e_room_keys.sql.postgres new file mode 100644 index 000000000000..9b98924165c3 --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_e2e_room_keys.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE e2e_room_keys REPLICA IDENTITY USING INDEX e2e_room_keys_with_version_idx; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_e2e_room_keys_versions.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_e2e_room_keys_versions.sql.postgres new file mode 100644 index 000000000000..ac7aaba44de9 --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_e2e_room_keys_versions.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE e2e_room_keys_versions REPLICA IDENTITY USING INDEX e2e_room_keys_versions_idx; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_erased_users.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_erased_users.sql.postgres new file mode 100644 index 000000000000..c775f81333cd --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_erased_users.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE erased_users REPLICA IDENTITY USING INDEX erased_users_user; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_event_auth.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_event_auth.sql.postgres new file mode 100644 index 000000000000..d8014d123bc8 --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_event_auth.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE event_auth REPLICA IDENTITY FULL; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_event_auth_chain_links.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_event_auth_chain_links.sql.postgres new file mode 100644 index 000000000000..c2f9c90d42e0 --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_event_auth_chain_links.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE event_auth_chain_links REPLICA IDENTITY FULL; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_event_backward_extremities.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_event_backward_extremities.sql.postgres new file mode 100644 index 000000000000..751541a929a9 --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_event_backward_extremities.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE event_backward_extremities REPLICA IDENTITY USING INDEX event_backward_extremities_event_id_room_id_key; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_event_edges.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_event_edges.sql.postgres new file mode 100644 index 000000000000..066ea763f1ef --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_event_edges.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE event_edges REPLICA IDENTITY USING INDEX event_edges_event_id_prev_event_id_idx; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_event_forward_extremities.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_event_forward_extremities.sql.postgres new file mode 100644 index 000000000000..d70d6fd55c57 --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_event_forward_extremities.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE event_forward_extremities REPLICA IDENTITY USING INDEX event_forward_extremities_event_id_room_id_key; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_event_json.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_event_json.sql.postgres new file mode 100644 index 000000000000..b614b9642292 --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_event_json.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE event_json REPLICA IDENTITY USING INDEX event_json_event_id_key; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_event_push_actions.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_event_push_actions.sql.postgres new file mode 100644 index 000000000000..7a92620270a8 --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_event_push_actions.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE event_push_actions REPLICA IDENTITY FULL; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_event_push_actions_staging.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_event_push_actions_staging.sql.postgres new file mode 100644 index 000000000000..292ed9787d8f --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_event_push_actions_staging.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE event_push_actions_staging REPLICA IDENTITY FULL; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_event_push_summary.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_event_push_summary.sql.postgres new file mode 100644 index 000000000000..42a9b6b6762a --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_event_push_summary.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE event_push_summary REPLICA IDENTITY FULL; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_event_push_summary_last_receipt_stream_id.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_event_push_summary_last_receipt_stream_id.sql.postgres new file mode 100644 index 000000000000..707e498a6ebf --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_event_push_summary_last_receipt_stream_id.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE event_push_summary_last_receipt_stream_id REPLICA IDENTITY USING INDEX event_push_summary_last_receipt_stream_id_lock_key; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_event_push_summary_stream_ordering.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_event_push_summary_stream_ordering.sql.postgres new file mode 100644 index 000000000000..1240e3e1b51d --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_event_push_summary_stream_ordering.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE event_push_summary_stream_ordering REPLICA IDENTITY USING INDEX event_push_summary_stream_ordering_lock_key; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_event_relations.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_event_relations.sql.postgres new file mode 100644 index 000000000000..6b62a6bc2003 --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_event_relations.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE event_relations REPLICA IDENTITY USING INDEX event_relations_id; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_event_search.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_event_search.sql.postgres new file mode 100644 index 000000000000..e8467fc3d37c --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_event_search.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE event_search REPLICA IDENTITY FULL; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_event_to_state_groups.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_event_to_state_groups.sql.postgres new file mode 100644 index 000000000000..4206975c3563 --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_event_to_state_groups.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE event_to_state_groups REPLICA IDENTITY USING INDEX event_to_state_groups_event_id_key; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_event_txn_id.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_event_txn_id.sql.postgres new file mode 100644 index 000000000000..a2bacbc96d57 --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_event_txn_id.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE event_txn_id REPLICA IDENTITY USING INDEX event_txn_id_event_id; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_event_txn_id_device_id.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_event_txn_id_device_id.sql.postgres new file mode 100644 index 000000000000..05fce1e6b094 --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_event_txn_id_device_id.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE event_txn_id_device_id REPLICA IDENTITY USING INDEX event_txn_id_device_id_event_id; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_events.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_events.sql.postgres new file mode 100644 index 000000000000..9046b8eb08b5 --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_events.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE events REPLICA IDENTITY USING INDEX events_event_id_key; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_federation_inbound_events_staging.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_federation_inbound_events_staging.sql.postgres new file mode 100644 index 000000000000..d9d2f88f675a --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_federation_inbound_events_staging.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE federation_inbound_events_staging REPLICA IDENTITY USING INDEX federation_inbound_events_staging_instance_event; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_federation_stream_position.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_federation_stream_position.sql.postgres new file mode 100644 index 000000000000..5f9df14d7dcf --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_federation_stream_position.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE federation_stream_position REPLICA IDENTITY USING INDEX federation_stream_position_instance; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_ignored_users.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_ignored_users.sql.postgres new file mode 100644 index 000000000000..dd082b7f9e3e --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_ignored_users.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE ignored_users REPLICA IDENTITY USING INDEX ignored_users_uniqueness; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_insertion_event_edges.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_insertion_event_edges.sql.postgres new file mode 100644 index 000000000000..8322722b1ab1 --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_insertion_event_edges.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE insertion_event_edges REPLICA IDENTITY FULL; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_insertion_event_extremities.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_insertion_event_extremities.sql.postgres new file mode 100644 index 000000000000..1816ce5a498d --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_insertion_event_extremities.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE insertion_event_extremities REPLICA IDENTITY USING INDEX insertion_event_extremities_event_id; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_insertion_events.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_insertion_events.sql.postgres new file mode 100644 index 000000000000..1fdbd597c7f1 --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_insertion_events.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE insertion_events REPLICA IDENTITY USING INDEX insertion_events_event_id; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_local_current_membership.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_local_current_membership.sql.postgres new file mode 100644 index 000000000000..f36bad701a5d --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_local_current_membership.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE local_current_membership REPLICA IDENTITY USING INDEX local_current_membership_idx; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_local_media_repository.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_local_media_repository.sql.postgres new file mode 100644 index 000000000000..78a5128af384 --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_local_media_repository.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE local_media_repository REPLICA IDENTITY FULL; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_local_media_repository_thumbnails.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_local_media_repository_thumbnails.sql.postgres new file mode 100644 index 000000000000..6cd53f8e15aa --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_local_media_repository_thumbnails.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE local_media_repository_thumbnails REPLICA IDENTITY FULL; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_local_media_repository_url_cache.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_local_media_repository_url_cache.sql.postgres new file mode 100644 index 000000000000..a41590ce6bee --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_local_media_repository_url_cache.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE local_media_repository_url_cache REPLICA IDENTITY FULL; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_monthly_active_users.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_monthly_active_users.sql.postgres new file mode 100644 index 000000000000..326fc279d3c0 --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_monthly_active_users.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE monthly_active_users REPLICA IDENTITY USING INDEX monthly_active_users_users; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_partial_state_events.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_partial_state_events.sql.postgres new file mode 100644 index 000000000000..c6acd84315be --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_partial_state_events.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE partial_state_events REPLICA IDENTITY USING INDEX partial_state_events_event_id_key; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_partial_state_rooms_servers.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_partial_state_rooms_servers.sql.postgres new file mode 100644 index 000000000000..c618660b5f50 --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_partial_state_rooms_servers.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE partial_state_rooms_servers REPLICA IDENTITY USING INDEX partial_state_rooms_servers_room_id_server_name_key; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_presence_stream.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_presence_stream.sql.postgres new file mode 100644 index 000000000000..4e5d8d7a68d4 --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_presence_stream.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE presence_stream REPLICA IDENTITY FULL; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_profiles.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_profiles.sql.postgres new file mode 100644 index 000000000000..bb25e7a67a12 --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_profiles.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE profiles REPLICA IDENTITY USING INDEX profiles_user_id_key; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_push_rules_stream.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_push_rules_stream.sql.postgres new file mode 100644 index 000000000000..f305d9a73a52 --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_push_rules_stream.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE push_rules_stream REPLICA IDENTITY FULL; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_ratelimit_override.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_ratelimit_override.sql.postgres new file mode 100644 index 000000000000..f10f359155bf --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_ratelimit_override.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE ratelimit_override REPLICA IDENTITY USING INDEX ratelimit_override_idx; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_receipts_graph.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_receipts_graph.sql.postgres new file mode 100644 index 000000000000..954dd9266771 --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_receipts_graph.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE receipts_graph REPLICA IDENTITY FULL; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_receipts_linearized.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_receipts_linearized.sql.postgres new file mode 100644 index 000000000000..302d88753189 --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_receipts_linearized.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE receipts_linearized REPLICA IDENTITY FULL; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_received_transactions.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_received_transactions.sql.postgres new file mode 100644 index 000000000000..3b8c2277172e --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_received_transactions.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE received_transactions REPLICA IDENTITY FULL; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_redactions.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_redactions.sql.postgres new file mode 100644 index 000000000000..ffa3444b955b --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_redactions.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE redactions REPLICA IDENTITY USING INDEX redactions_event_id_key; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_registration_tokens.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_registration_tokens.sql.postgres new file mode 100644 index 000000000000..3fdb5bbd54a1 --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_registration_tokens.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE registration_tokens REPLICA IDENTITY USING INDEX registration_tokens_token_key; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_rejections.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_rejections.sql.postgres new file mode 100644 index 000000000000..c7a7218578d8 --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_rejections.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE rejections REPLICA IDENTITY USING INDEX rejections_event_id_key; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_remote_media_cache.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_remote_media_cache.sql.postgres new file mode 100644 index 000000000000..d0ecfe1a759b --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_remote_media_cache.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE remote_media_cache REPLICA IDENTITY FULL; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_remote_media_cache_thumbnails.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_remote_media_cache_thumbnails.sql.postgres new file mode 100644 index 000000000000..9d7b63dfb42a --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_remote_media_cache_thumbnails.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE remote_media_cache_thumbnails REPLICA IDENTITY FULL; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_room_account_data.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_room_account_data.sql.postgres new file mode 100644 index 000000000000..36e756f6b198 --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_room_account_data.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE room_account_data REPLICA IDENTITY USING INDEX room_account_data_uniqueness; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_room_alias_servers.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_room_alias_servers.sql.postgres new file mode 100644 index 000000000000..df550076ac5c --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_room_alias_servers.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE room_alias_servers REPLICA IDENTITY FULL; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_room_aliases.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_room_aliases.sql.postgres new file mode 100644 index 000000000000..bfe8db5375d8 --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_room_aliases.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE room_aliases REPLICA IDENTITY USING INDEX room_aliases_room_alias_key; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_room_depth.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_room_depth.sql.postgres new file mode 100644 index 000000000000..77c78ee4ab49 --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_room_depth.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE room_depth REPLICA IDENTITY USING INDEX room_depth_room_id_key; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_room_forgetter_stream_pos.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_room_forgetter_stream_pos.sql.postgres new file mode 100644 index 000000000000..51de67b54947 --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_room_forgetter_stream_pos.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE room_forgetter_stream_pos REPLICA IDENTITY USING INDEX room_forgetter_stream_pos_lock_key; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_room_memberships.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_room_memberships.sql.postgres new file mode 100644 index 000000000000..5819409058d1 --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_room_memberships.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE room_memberships REPLICA IDENTITY USING INDEX room_memberships_event_id_key; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_room_stats_earliest_token.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_room_stats_earliest_token.sql.postgres new file mode 100644 index 000000000000..bacd9587d28f --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_room_stats_earliest_token.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE room_stats_earliest_token REPLICA IDENTITY USING INDEX room_stats_earliest_token_idx; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_room_stats_state.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_room_stats_state.sql.postgres new file mode 100644 index 000000000000..a64db05f4471 --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_room_stats_state.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE room_stats_state REPLICA IDENTITY USING INDEX room_stats_state_room; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_room_tags.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_room_tags.sql.postgres new file mode 100644 index 000000000000..cc6970475189 --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_room_tags.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE room_tags REPLICA IDENTITY USING INDEX room_tag_uniqueness; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_room_tags_revisions.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_room_tags_revisions.sql.postgres new file mode 100644 index 000000000000..d31063277c9e --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_room_tags_revisions.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE room_tags_revisions REPLICA IDENTITY USING INDEX room_tag_revisions_uniqueness; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_server_keys_json.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_server_keys_json.sql.postgres new file mode 100644 index 000000000000..9c0bfb5623c2 --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_server_keys_json.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE server_keys_json REPLICA IDENTITY USING INDEX server_keys_json_uniqueness; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_server_signature_keys.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_server_signature_keys.sql.postgres new file mode 100644 index 000000000000..1b3d5e68e5b6 --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_server_signature_keys.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE server_signature_keys REPLICA IDENTITY FULL; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_sessions.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_sessions.sql.postgres new file mode 100644 index 000000000000..eb8d621795b0 --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_sessions.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE sessions REPLICA IDENTITY USING INDEX sessions_session_type_session_id_key; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_state_events.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_state_events.sql.postgres new file mode 100644 index 000000000000..7e23c8a0545a --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_state_events.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE state_events REPLICA IDENTITY USING INDEX state_events_event_id_key; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_stats_incremental_position.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_stats_incremental_position.sql.postgres new file mode 100644 index 000000000000..a4a700a3a408 --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_stats_incremental_position.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE stats_incremental_position REPLICA IDENTITY USING INDEX stats_incremental_position_lock_key; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_stream_ordering_to_exterm.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_stream_ordering_to_exterm.sql.postgres new file mode 100644 index 000000000000..a70074189610 --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_stream_ordering_to_exterm.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE stream_ordering_to_exterm REPLICA IDENTITY FULL; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_stream_positions.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_stream_positions.sql.postgres new file mode 100644 index 000000000000..83ff1c382542 --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_stream_positions.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE stream_positions REPLICA IDENTITY USING INDEX stream_positions_idx; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_threads.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_threads.sql.postgres new file mode 100644 index 000000000000..a3891e4208e8 --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_threads.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE threads REPLICA IDENTITY USING INDEX threads_uniqueness; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_threepid_guest_access_tokens.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_threepid_guest_access_tokens.sql.postgres new file mode 100644 index 000000000000..df31cf4f1d6c --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_threepid_guest_access_tokens.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE threepid_guest_access_tokens REPLICA IDENTITY FULL; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_timeline_gaps.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_timeline_gaps.sql.postgres new file mode 100644 index 000000000000..8fb41226c14c --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_timeline_gaps.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE timeline_gaps REPLICA IDENTITY FULL; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_ui_auth_sessions.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_ui_auth_sessions.sql.postgres new file mode 100644 index 000000000000..5d2af97f729b --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_ui_auth_sessions.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE ui_auth_sessions REPLICA IDENTITY USING INDEX ui_auth_sessions_session_id_key; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_ui_auth_sessions_credentials.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_ui_auth_sessions_credentials.sql.postgres new file mode 100644 index 000000000000..efb2359faa7e --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_ui_auth_sessions_credentials.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE ui_auth_sessions_credentials REPLICA IDENTITY USING INDEX ui_auth_sessions_credentials_session_id_stage_type_key; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_ui_auth_sessions_ips.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_ui_auth_sessions_ips.sql.postgres new file mode 100644 index 000000000000..a4b86eaf0719 --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_ui_auth_sessions_ips.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE ui_auth_sessions_ips REPLICA IDENTITY USING INDEX ui_auth_sessions_ips_session_id_ip_user_agent_key; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_user_daily_visits.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_user_daily_visits.sql.postgres new file mode 100644 index 000000000000..bb22ec5335c1 --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_user_daily_visits.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE user_daily_visits REPLICA IDENTITY FULL; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_user_directory.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_user_directory.sql.postgres new file mode 100644 index 000000000000..947f8c5c0fa6 --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_user_directory.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE user_directory REPLICA IDENTITY USING INDEX user_directory_user_idx; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_user_directory_search.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_user_directory_search.sql.postgres new file mode 100644 index 000000000000..d4217799440a --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_user_directory_search.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE user_directory_search REPLICA IDENTITY USING INDEX user_directory_search_user_idx; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_user_directory_stream_pos.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_user_directory_stream_pos.sql.postgres new file mode 100644 index 000000000000..59201750b0e5 --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_user_directory_stream_pos.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE user_directory_stream_pos REPLICA IDENTITY USING INDEX user_directory_stream_pos_lock_key; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_user_external_ids.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_user_external_ids.sql.postgres new file mode 100644 index 000000000000..219ee16ddea6 --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_user_external_ids.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE user_external_ids REPLICA IDENTITY USING INDEX user_external_ids_auth_provider_external_id_key; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_user_filters.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_user_filters.sql.postgres new file mode 100644 index 000000000000..5d3e0ae475be --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_user_filters.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE user_filters REPLICA IDENTITY FULL; -- sadly the `CHECK` constraint is not enough here diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_user_ips.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_user_ips.sql.postgres new file mode 100644 index 000000000000..5a1d8770d0c3 --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_user_ips.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE user_ips REPLICA IDENTITY USING INDEX user_ips_user_token_ip_unique_index; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_user_signature_stream.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_user_signature_stream.sql.postgres new file mode 100644 index 000000000000..8bc27df00a2b --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_user_signature_stream.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE user_signature_stream REPLICA IDENTITY USING INDEX user_signature_stream_idx; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_user_threepid_id_server.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_user_threepid_id_server.sql.postgres new file mode 100644 index 000000000000..a0ba8e67731a --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_user_threepid_id_server.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE user_threepid_id_server REPLICA IDENTITY USING INDEX user_threepid_id_server_idx; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_user_threepids.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_user_threepids.sql.postgres new file mode 100644 index 000000000000..9c70c9011e50 --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_user_threepids.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE user_threepids REPLICA IDENTITY USING INDEX medium_address; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_users.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_users.sql.postgres new file mode 100644 index 000000000000..725e6cdc7667 --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_users.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE users REPLICA IDENTITY FULL; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_users_in_public_rooms.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_users_in_public_rooms.sql.postgres new file mode 100644 index 000000000000..86988a55505f --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_users_in_public_rooms.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE users_in_public_rooms REPLICA IDENTITY USING INDEX users_in_public_rooms_u_idx; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_users_pending_deactivation.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_users_pending_deactivation.sql.postgres new file mode 100644 index 000000000000..7e72946d14a1 --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_users_pending_deactivation.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE users_pending_deactivation REPLICA IDENTITY FULL; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_users_who_share_private_rooms.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_users_who_share_private_rooms.sql.postgres new file mode 100644 index 000000000000..6c7213920aa7 --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_users_who_share_private_rooms.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE users_who_share_private_rooms REPLICA IDENTITY USING INDEX users_who_share_private_rooms_u_idx; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_worker_locks.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_worker_locks.sql.postgres new file mode 100644 index 000000000000..1bba45e05c47 --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_worker_locks.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE worker_locks REPLICA IDENTITY USING INDEX worker_locks_key; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_worker_read_write_locks.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_worker_read_write_locks.sql.postgres new file mode 100644 index 000000000000..b9d35eb4f693 --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_worker_read_write_locks.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE worker_read_write_locks REPLICA IDENTITY USING INDEX worker_read_write_locks_key; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_worker_read_write_locks_mode.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_worker_read_write_locks_mode.sql.postgres new file mode 100644 index 000000000000..5871c98deeee --- /dev/null +++ b/synapse/storage/schema/main/delta/83/10_replica_identity_worker_read_write_locks_mode.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE worker_read_write_locks_mode REPLICA IDENTITY USING INDEX worker_read_write_locks_mode_key; diff --git a/synapse/storage/schema/state/delta/83/10_replica_identity_state_group_edges.sql.postgres b/synapse/storage/schema/state/delta/83/10_replica_identity_state_group_edges.sql.postgres new file mode 100644 index 000000000000..aa8fc2514cb6 --- /dev/null +++ b/synapse/storage/schema/state/delta/83/10_replica_identity_state_group_edges.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE state_group_edges REPLICA IDENTITY USING INDEX state_group_edges_unique_idx; diff --git a/synapse/storage/schema/state/delta/83/10_replica_identity_state_groups_state.sql.postgres b/synapse/storage/schema/state/delta/83/10_replica_identity_state_groups_state.sql.postgres new file mode 100644 index 000000000000..438603197724 --- /dev/null +++ b/synapse/storage/schema/state/delta/83/10_replica_identity_state_groups_state.sql.postgres @@ -0,0 +1 @@ +ALTER TABLE state_groups_state REPLICA IDENTITY FULL; diff --git a/tests/storage/test_database.py b/tests/storage/test_database.py index 4d0ebb550d29..aa8c76f18777 100644 --- a/tests/storage/test_database.py +++ b/tests/storage/test_database.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Callable, Tuple +from typing import Callable, List, Tuple from unittest.mock import Mock, call from twisted.internet import defer @@ -29,6 +29,7 @@ from synapse.util import Clock from tests import unittest +from tests.utils import USE_POSTGRES_FOR_TESTS class TupleComparisonClauseTestCase(unittest.TestCase): @@ -279,3 +280,85 @@ def _test_txn(txn: LoggingTransaction) -> None: ] ) self.assertEqual(exception_callback.call_count, 6) # no additional calls + + +class PostgresReplicaIdentityTestCase(unittest.HomeserverTestCase): + if not USE_POSTGRES_FOR_TESTS: + skip = "Requires Postgres" + + def prepare( + self, reactor: MemoryReactor, clock: Clock, homeserver: HomeServer + ) -> None: + self.db_pools = homeserver.get_datastores().databases + + def test_all_tables_have_postgres_replica_identity(self) -> None: + """ + Tests that all tables have a Postgres REPLICA IDENTITY. + (See https://github.com/matrix-org/synapse/issues/16224). + + Tables with a PRIMARY KEY have an implied REPLICA IDENTITY and are fine. + Other tables need them to be set with `ALTER TABLE`. + + A REPLICA IDENTITY is required for Postgres logical replication to work + properly without blocking updates and deletes. + """ + + sql = """ + -- Select tables that have no primary key and use the default replica identity rule + -- (the default is to use the primary key) + WITH tables_no_pkey AS ( + SELECT tbl.table_schema, tbl.table_name + FROM information_schema.tables tbl + WHERE table_type = 'BASE TABLE' + AND table_schema not in ('pg_catalog', 'information_schema') + AND NOT EXISTS ( + SELECT 1 + FROM information_schema.table_constraints tc + WHERE tc.constraint_type = 'PRIMARY KEY' + AND tc.table_schema = tbl.table_schema + AND tc.table_name = tbl.table_name + ) + ) + SELECT pg_class.oid::regclass FROM tables_no_pkey INNER JOIN pg_class ON pg_class.oid::regclass = table_name::regclass + WHERE relreplident = 'd' + + UNION + + -- Also select tables that use an index as a replica identity + -- but where the index doesn't exist + -- (e.g. it could have been deleted) + SELECT pg_class.oid::regclass + FROM information_schema.tables tbl + INNER JOIN pg_class ON pg_class.oid::regclass = table_name::regclass + WHERE table_type = 'BASE TABLE' + AND table_schema not in ('pg_catalog', 'information_schema') + + -- 'i' means an index is used as the replica identity + AND relreplident = 'i' + + -- look for indices that are marked as the replica identity + AND NOT EXISTS ( + SELECT indexrelid::regclass + FROM pg_index + WHERE indrelid = pg_class.oid::regclass AND indisreplident + ) + """ + + def _list_tables_with_missing_replica_identities_txn( + txn: LoggingTransaction, + ) -> List[str]: + txn.execute(sql) + return [table_name for table_name, in txn] + + for pool in self.db_pools: + missing = self.get_success( + pool.runInteraction( + "test_list_missing_replica_identities", + _list_tables_with_missing_replica_identities_txn, + ) + ) + self.assertEqual( + len(missing), + 0, + f"The following tables in the {pool.name()!r} database are missing REPLICA IDENTITIES: {missing!r}.", + ) From 0bb8e418a41c6f583ca9d705b400e37e2308a534 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Tue, 5 Dec 2023 11:08:40 +0000 Subject: [PATCH 22/29] Fix postgres schema after dropping old tables (#16730) --- changelog.d/16730.misc | 1 + .../main/delta/83/10_replica_identity_batch_events.sql.postgres | 1 - .../main/delta/83/10_replica_identity_event_txn_id.sql.postgres | 1 - .../83/10_replica_identity_insertion_event_edges.sql.postgres | 1 - .../10_replica_identity_insertion_event_extremities.sql.postgres | 1 - .../delta/83/10_replica_identity_insertion_events.sql.postgres | 1 - 6 files changed, 1 insertion(+), 5 deletions(-) create mode 100644 changelog.d/16730.misc delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_batch_events.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_event_txn_id.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_insertion_event_edges.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_insertion_event_extremities.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_insertion_events.sql.postgres diff --git a/changelog.d/16730.misc b/changelog.d/16730.misc new file mode 100644 index 000000000000..baee042f2f2c --- /dev/null +++ b/changelog.d/16730.misc @@ -0,0 +1 @@ +Add a Postgres `REPLICA IDENTITY` to tables that do not have an implicit one. This should allow use of Postgres logical replication. \ No newline at end of file diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_batch_events.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_batch_events.sql.postgres deleted file mode 100644 index cd4471f4d7c7..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_batch_events.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE batch_events REPLICA IDENTITY USING INDEX chunk_events_event_id; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_event_txn_id.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_event_txn_id.sql.postgres deleted file mode 100644 index a2bacbc96d57..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_event_txn_id.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE event_txn_id REPLICA IDENTITY USING INDEX event_txn_id_event_id; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_insertion_event_edges.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_insertion_event_edges.sql.postgres deleted file mode 100644 index 8322722b1ab1..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_insertion_event_edges.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE insertion_event_edges REPLICA IDENTITY FULL; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_insertion_event_extremities.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_insertion_event_extremities.sql.postgres deleted file mode 100644 index 1816ce5a498d..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_insertion_event_extremities.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE insertion_event_extremities REPLICA IDENTITY USING INDEX insertion_event_extremities_event_id; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_insertion_events.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_insertion_events.sql.postgres deleted file mode 100644 index 1fdbd597c7f1..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_insertion_events.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE insertion_events REPLICA IDENTITY USING INDEX insertion_events_event_id; From ea783550bb8098f7c37e6ebfa83902108f377890 Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Tue, 5 Dec 2023 11:40:27 +0000 Subject: [PATCH 23/29] Set response values to zero if None for /_synapse/admin/v1/federation/destinations (#16729) --- changelog.d/16729.bugfix | 1 + synapse/rest/admin/federation.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 changelog.d/16729.bugfix diff --git a/changelog.d/16729.bugfix b/changelog.d/16729.bugfix new file mode 100644 index 000000000000..50e72ac1f817 --- /dev/null +++ b/changelog.d/16729.bugfix @@ -0,0 +1 @@ +Fix GET /_synapse/admin/v1/federation/destinations returning null (instead of 0) for `retry_last_ts` and `retry_interval`. diff --git a/synapse/rest/admin/federation.py b/synapse/rest/admin/federation.py index a6ce787da1af..2f238f021776 100644 --- a/synapse/rest/admin/federation.py +++ b/synapse/rest/admin/federation.py @@ -89,8 +89,8 @@ async def on_GET(self, request: SynapseRequest) -> Tuple[int, JsonDict]: "destinations": [ { "destination": r[0], - "retry_last_ts": r[1], - "retry_interval": r[2], + "retry_last_ts": r[1] or 0, + "retry_interval": r[2] or 0, "failure_ts": r[3], "last_successful_stream_ordering": r[4], } From e87499b3f46a31c25df2da8686d3b913f8094660 Mon Sep 17 00:00:00 2001 From: "Amanda H. L. de Andrade Katz" Date: Tue, 5 Dec 2023 08:42:56 -0300 Subject: [PATCH 24/29] Add how to validate configuration file with synapse.config script (#16714) --- changelog.d/16714.doc | 1 + .../usage/configuration/config_documentation.md | 17 +++++++++++++++++ 2 files changed, 18 insertions(+) create mode 100644 changelog.d/16714.doc diff --git a/changelog.d/16714.doc b/changelog.d/16714.doc new file mode 100644 index 000000000000..511d2b60224d --- /dev/null +++ b/changelog.d/16714.doc @@ -0,0 +1 @@ +Add documentation for how to validate the configuration file with synapse.config script. diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index e6fcffb5706b..425ec75542d3 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -33,6 +33,23 @@ In addition, configuration options referring to size use the following suffixes: For example, setting `max_avatar_size: 10M` means that Synapse will not accept files larger than 10,485,760 bytes for a user avatar. +## Config Validation + +The configuration file can be validated with the following command: +```bash +python -m synapse.config read -c +``` + +To validate the entire file, omit `read `: +```bash +python -m synapse.config -c +``` + +To see how to set other options, check the help reference: +```bash +python -m synapse.config --help +``` + ### YAML The configuration file is a [YAML](https://yaml.org/) file, which means that certain syntax rules apply if you want your config file to be read properly. A few helpful things to know: From 0a00c99823818228479f9fdb8c77e61a49a65e18 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Tue, 5 Dec 2023 12:06:21 +0000 Subject: [PATCH 25/29] Fix upgrading a room without `events` field in power levels (#16725) --- changelog.d/16725.bugfix | 1 + synapse/handlers/room.py | 2 +- tests/rest/client/test_upgrade_room.py | 28 ++++++++++++++++++++++++++ 3 files changed, 30 insertions(+), 1 deletion(-) create mode 100644 changelog.d/16725.bugfix diff --git a/changelog.d/16725.bugfix b/changelog.d/16725.bugfix new file mode 100644 index 000000000000..42dc249fb9df --- /dev/null +++ b/changelog.d/16725.bugfix @@ -0,0 +1 @@ +Fix a bug introduced in Synapse 1.7.2 where rooms whose power levels lacked an `events` field could not be upgraded. diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index f865bed1ec2c..2823ca6f0d05 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -549,7 +549,7 @@ async def clone_existing_room( except (TypeError, ValueError): ban = 50 needed_power_level = max( - state_default_int, ban, max(event_power_levels.values()) + state_default_int, ban, max(event_power_levels.values(), default=0) ) # Get the user's current power level, this matches the logic in get_user_power_level, diff --git a/tests/rest/client/test_upgrade_room.py b/tests/rest/client/test_upgrade_room.py index 0b4c691318e9..28f772261b72 100644 --- a/tests/rest/client/test_upgrade_room.py +++ b/tests/rest/client/test_upgrade_room.py @@ -246,6 +246,34 @@ def test_stringy_power_levels(self) -> None: # We should now have an integer power level. self.assertEqual(new_power_levels["users"][self.creator], 100, new_power_levels) + def test_events_field_missing(self) -> None: + """Regression test for https://github.com/matrix-org/synapse/issues/16715.""" + # Create a new room. + room_id = self.helper.create_room_as( + self.creator, tok=self.creator_token, room_version="10" + ) + self.helper.join(room_id, self.other, tok=self.other_token) + + # Retrieve the room's current power levels. + power_levels = self.helper.get_state( + room_id, + "m.room.power_levels", + tok=self.creator_token, + ) + + # Remove the events field and re-set the power levels. + del power_levels["events"] + self.helper.send_state( + room_id, + "m.room.power_levels", + body=power_levels, + tok=self.creator_token, + ) + + # Upgrade the room. Check the homeserver reports success. + channel = self._upgrade_room(room_id=room_id) + self.assertEqual(200, channel.code, channel.result) + def test_space(self) -> None: """Test upgrading a space.""" From 3c83d8f0afcaa8461516321d49a64906e0ec865b Mon Sep 17 00:00:00 2001 From: David Robertson Date: Tue, 5 Dec 2023 13:14:36 +0000 Subject: [PATCH 26/29] 1.98.0rc1 --- CHANGES.md | 74 +++++++++++++++++++++++++++++++++++++++ changelog.d/15207.feature | 1 - changelog.d/16522.misc | 1 - changelog.d/16552.misc | 1 - changelog.d/16636.feature | 1 - changelog.d/16642.bugfix | 1 - changelog.d/16658.misc | 1 - changelog.d/16661.doc | 1 - changelog.d/16667.misc | 1 - changelog.d/16668.misc | 1 - changelog.d/16670.bugfix | 1 - changelog.d/16672.feature | 1 - changelog.d/16673.misc | 1 - changelog.d/16677.misc | 1 - changelog.d/16682.misc | 1 - changelog.d/16684.misc | 1 - changelog.d/16695.doc | 1 - changelog.d/16697.misc | 1 - changelog.d/16699.feature | 1 - changelog.d/16700.doc | 1 - changelog.d/16701.feature | 1 - changelog.d/16702.misc | 1 - changelog.d/16704.misc | 1 - changelog.d/16707.feature | 1 - changelog.d/16714.doc | 1 - changelog.d/16725.bugfix | 1 - changelog.d/16729.bugfix | 1 - changelog.d/16730.misc | 1 - debian/changelog | 6 ++++ pyproject.toml | 2 +- 30 files changed, 81 insertions(+), 28 deletions(-) delete mode 100644 changelog.d/15207.feature delete mode 100644 changelog.d/16522.misc delete mode 100644 changelog.d/16552.misc delete mode 100644 changelog.d/16636.feature delete mode 100644 changelog.d/16642.bugfix delete mode 100644 changelog.d/16658.misc delete mode 100644 changelog.d/16661.doc delete mode 100644 changelog.d/16667.misc delete mode 100644 changelog.d/16668.misc delete mode 100644 changelog.d/16670.bugfix delete mode 100644 changelog.d/16672.feature delete mode 100644 changelog.d/16673.misc delete mode 100644 changelog.d/16677.misc delete mode 100644 changelog.d/16682.misc delete mode 100644 changelog.d/16684.misc delete mode 100644 changelog.d/16695.doc delete mode 100644 changelog.d/16697.misc delete mode 100644 changelog.d/16699.feature delete mode 100644 changelog.d/16700.doc delete mode 100644 changelog.d/16701.feature delete mode 100644 changelog.d/16702.misc delete mode 100644 changelog.d/16704.misc delete mode 100644 changelog.d/16707.feature delete mode 100644 changelog.d/16714.doc delete mode 100644 changelog.d/16725.bugfix delete mode 100644 changelog.d/16729.bugfix delete mode 100644 changelog.d/16730.misc diff --git a/CHANGES.md b/CHANGES.md index bb34267af331..c94736ea9a4b 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,77 @@ +# Synapse 1.98.0rc1 (2023-12-05) + +Synapse 1.98.0 will be the last Synapse release in 2023; the regular release cadence will resume in January 2024. + +Synapse will soon be forked by Element under an AGPLv3.0 licence (with CLA, for +proprietary dual licensing). You can read more about this here: + + - https://matrix.org/blog/2023/11/06/future-of-synapse-dendrite/ + - https://element.io/blog/element-to-adopt-agplv3/ + +The Matrix.org Foundation copy of the project will be archived. Any changes needed +by server administrators will be communicated via our usual announcements channels, +but we are striving to make this as seamless as possible. + +### Features + +- Synapse now declares support for Matrix v1.7, v1.8, and v1.9. ([\#16707](https://github.com/matrix-org/synapse/issues/16707)) +- Add `on_user_login` [module API](https://matrix-org.github.io/synapse/latest/modules/writing_a_module.html) callback allowing to execute custom code after (on) Auth. ([\#15207](https://github.com/matrix-org/synapse/issues/15207)) +- Support [MSC4069: Inhibit profile propagation](https://github.com/matrix-org/matrix-spec-proposals/pull/4069). ([\#16636](https://github.com/matrix-org/synapse/issues/16636)) +- Restore tracking of requests and monthly active users when delegating authentication via [MSC3861](https://github.com/matrix-org/synapse/pull/16672) to an OIDC provider. ([\#16672](https://github.com/matrix-org/synapse/issues/16672)) +- Add an autojoin setting for server notices rooms, so users may be joined directly instead of receiving an invite. ([\#16699](https://github.com/matrix-org/synapse/issues/16699)) +- Follow redirects when downloading media over federation (per [MSC3860](https://github.com/matrix-org/matrix-spec-proposals/pull/3860)). ([\#16701](https://github.com/matrix-org/synapse/issues/16701)) + +### Bugfixes + +- Enable refreshable tokens on the admin registration endpoint. ([\#16642](https://github.com/matrix-org/synapse/issues/16642)) +- Consistently bypass rate limits when using the server notice admin API. ([\#16670](https://github.com/matrix-org/synapse/issues/16670)) +- Fix a bug introduced in Synapse 1.7.2 where rooms whose power levels lacked an `events` field could not be upgraded. ([\#16725](https://github.com/matrix-org/synapse/issues/16725)) +- Fix `GET /_synapse/admin/v1/federation/destinations` [admin API](https://matrix-org.github.io/synapse/latest/usage/administration/admin_api/index.html) returning null (instead of 0) for `retry_last_ts` and `retry_interval`. ([\#16729](https://github.com/matrix-org/synapse/issues/16729)) + +### Improved Documentation + +- Add schema rollback information to documentation. ([\#16661](https://github.com/matrix-org/synapse/issues/16661)) +- Fix poetry version typo in [contributors' guide](https://matrix-org.github.io/synapse/latest/development/contributing_guide.html). ([\#16695](https://github.com/matrix-org/synapse/issues/16695)) +- Switch the example UNIX socket paths to `/run`. Add HAProxy example configuration for UNIX sockets. ([\#16700](https://github.com/matrix-org/synapse/issues/16700)) +- Add documentation for how to validate the configuration file with `synapse.config` script. ([\#16714](https://github.com/matrix-org/synapse/issues/16714)) + +### Internal Changes + +- Clean-up unused tables. ([\#16522](https://github.com/matrix-org/synapse/issues/16522)) +- Reduce a little database load while processing state auth chains. ([\#16552](https://github.com/matrix-org/synapse/issues/16552)) +- Add a Postgres `REPLICA IDENTITY` to tables that do not have an implicit one. This should allow use of Postgres logical replication. ([\#16658](https://github.com/matrix-org/synapse/issues/16658), [\#16730](https://github.com/matrix-org/synapse/issues/16730)) +- Reduce database load of pruning old `user_ips`. ([\#16667](https://github.com/matrix-org/synapse/issues/16667)) +- Reduce DB load when forget on leave setting is disabled. ([\#16668](https://github.com/matrix-org/synapse/issues/16668)) +- Ignore `encryption_enabled_by_default_for_room_type` setting when creating server notices room, since the notices will be send unencrypted anyway. ([\#16677](https://github.com/matrix-org/synapse/issues/16677)) +- Correctly read the to-device stream ID on startup using SQLite. ([\#16682](https://github.com/matrix-org/synapse/issues/16682)) +- Reoranganise test files. ([\#16684](https://github.com/matrix-org/synapse/issues/16684)) +- Remove old full schema dumps which are no longer used. ([\#16697](https://github.com/matrix-org/synapse/issues/16697)) +- Raise poetry-core upper bound to <=1.8.1. This allows contributors to import Synapse after `poetry install`ing with Poetry 1.6 and above. Contributed by Mo Balaa. ([\#16702](https://github.com/matrix-org/synapse/issues/16702)) +- Add a workflow to try and automatically fixup linting in a PR. ([\#16704](https://github.com/matrix-org/synapse/issues/16704)) + + +### Updates to locked dependencies + +* Bump cryptography from 41.0.5 to 41.0.6. ([\#16703](https://github.com/matrix-org/synapse/issues/16703)) +* Bump cryptography from 41.0.6 to 41.0.7. ([\#16721](https://github.com/matrix-org/synapse/issues/16721)) +* Bump idna from 3.4 to 3.6. ([\#16720](https://github.com/matrix-org/synapse/issues/16720)) +* Bump jsonschema from 4.19.1 to 4.20.0. ([\#16692](https://github.com/matrix-org/synapse/issues/16692)) +* Bump matrix-org/netlify-pr-preview from 2 to 3. ([\#16719](https://github.com/matrix-org/synapse/issues/16719)) +* Bump phonenumbers from 8.13.23 to 8.13.26. ([\#16722](https://github.com/matrix-org/synapse/issues/16722)) +* Bump prometheus-client from 0.18.0 to 0.19.0. ([\#16691](https://github.com/matrix-org/synapse/issues/16691)) +* Bump pyasn1 from 0.5.0 to 0.5.1. ([\#16689](https://github.com/matrix-org/synapse/issues/16689)) +* Bump pydantic from 2.4.2 to 2.5.1. ([\#16663](https://github.com/matrix-org/synapse/issues/16663)) +- Bump pyo3 (0.19.2→0.20.0), pythonize (0.19.0→0.20.0) and pyo3-log (0.8.1→0.9.0). ([\#16673](https://github.com/matrix-org/synapse/issues/16673)) +* Bump pyopenssl from 23.2.0 to 23.3.0. ([\#16662](https://github.com/matrix-org/synapse/issues/16662)) +* Bump ruff from 0.1.4 to 0.1.6. ([\#16690](https://github.com/matrix-org/synapse/issues/16690)) +* Bump sentry-sdk from 1.32.0 to 1.35.0. ([\#16666](https://github.com/matrix-org/synapse/issues/16666)) +* Bump serde from 1.0.192 to 1.0.193. ([\#16693](https://github.com/matrix-org/synapse/issues/16693)) +* Bump sphinx-autodoc2 from 0.4.2 to 0.5.0. ([\#16723](https://github.com/matrix-org/synapse/issues/16723)) +* Bump types-jsonschema from 4.19.0.4 to 4.20.0.0. ([\#16724](https://github.com/matrix-org/synapse/issues/16724)) +* Bump types-pillow from 10.1.0.0 to 10.1.0.2. ([\#16664](https://github.com/matrix-org/synapse/issues/16664)) +* Bump types-psycopg2 from 2.9.21.15 to 2.9.21.16. ([\#16665](https://github.com/matrix-org/synapse/issues/16665)) +* Bump types-setuptools from 68.2.0.0 to 68.2.0.2. ([\#16688](https://github.com/matrix-org/synapse/issues/16688)) + # Synapse 1.97.0 (2023-11-28) Synapse will soon be forked by Element under an AGPLv3.0 licence (with CLA, for diff --git a/changelog.d/15207.feature b/changelog.d/15207.feature deleted file mode 100644 index 17790d62eb09..000000000000 --- a/changelog.d/15207.feature +++ /dev/null @@ -1 +0,0 @@ -Adds on_user_login ModuleAPI callback allowing to execute custom code after (on) Auth. \ No newline at end of file diff --git a/changelog.d/16522.misc b/changelog.d/16522.misc deleted file mode 100644 index 26059b108e6a..000000000000 --- a/changelog.d/16522.misc +++ /dev/null @@ -1 +0,0 @@ -Clean-up unused tables. diff --git a/changelog.d/16552.misc b/changelog.d/16552.misc deleted file mode 100644 index 73a0376df097..000000000000 --- a/changelog.d/16552.misc +++ /dev/null @@ -1 +0,0 @@ -Reduce a little database load while processing state auth chains. diff --git a/changelog.d/16636.feature b/changelog.d/16636.feature deleted file mode 100644 index a363eaafaf13..000000000000 --- a/changelog.d/16636.feature +++ /dev/null @@ -1 +0,0 @@ -Support MSC4069: Inhibit profile propagation. \ No newline at end of file diff --git a/changelog.d/16642.bugfix b/changelog.d/16642.bugfix deleted file mode 100644 index e83c8b4f9d58..000000000000 --- a/changelog.d/16642.bugfix +++ /dev/null @@ -1 +0,0 @@ -Enable refreshable tokens on the admin registration endpoint. diff --git a/changelog.d/16658.misc b/changelog.d/16658.misc deleted file mode 100644 index baee042f2f2c..000000000000 --- a/changelog.d/16658.misc +++ /dev/null @@ -1 +0,0 @@ -Add a Postgres `REPLICA IDENTITY` to tables that do not have an implicit one. This should allow use of Postgres logical replication. \ No newline at end of file diff --git a/changelog.d/16661.doc b/changelog.d/16661.doc deleted file mode 100644 index 74f8fc84b8f7..000000000000 --- a/changelog.d/16661.doc +++ /dev/null @@ -1 +0,0 @@ -Add schema rollback information to documentation. diff --git a/changelog.d/16667.misc b/changelog.d/16667.misc deleted file mode 100644 index 51aeca924397..000000000000 --- a/changelog.d/16667.misc +++ /dev/null @@ -1 +0,0 @@ -Reduce database load of pruning old `user_ips`. diff --git a/changelog.d/16668.misc b/changelog.d/16668.misc deleted file mode 100644 index 9ed004d6e46a..000000000000 --- a/changelog.d/16668.misc +++ /dev/null @@ -1 +0,0 @@ -Reduce DB load when forget on leave setting is disabled. diff --git a/changelog.d/16670.bugfix b/changelog.d/16670.bugfix deleted file mode 100644 index f1369abc0607..000000000000 --- a/changelog.d/16670.bugfix +++ /dev/null @@ -1 +0,0 @@ -Consistently bypass rate limits when using the server notice admin API. diff --git a/changelog.d/16672.feature b/changelog.d/16672.feature deleted file mode 100644 index 05ecf2220733..000000000000 --- a/changelog.d/16672.feature +++ /dev/null @@ -1 +0,0 @@ -Restore tracking of requests and monthly active users when delegating authentication to an [MSC3861](https://github.com/matrix-org/synapse/pull/16672) OIDC provider. diff --git a/changelog.d/16673.misc b/changelog.d/16673.misc deleted file mode 100644 index 8b274ede71f8..000000000000 --- a/changelog.d/16673.misc +++ /dev/null @@ -1 +0,0 @@ -Bump pyo3 (0.19.2→0.20.0), pythonize (0.19.0→0.20.0) and pyo3-log (0.8.1→0.9.0). diff --git a/changelog.d/16677.misc b/changelog.d/16677.misc deleted file mode 100644 index 20c37851c396..000000000000 --- a/changelog.d/16677.misc +++ /dev/null @@ -1 +0,0 @@ -Ignore `encryption_enabled_by_default_for_room_type` setting when creating server notices room, since the notices will be send unencrypted anyway. diff --git a/changelog.d/16682.misc b/changelog.d/16682.misc deleted file mode 100644 index 071715e83a80..000000000000 --- a/changelog.d/16682.misc +++ /dev/null @@ -1 +0,0 @@ -Correctly read the to-device stream ID on startup using SQLite. diff --git a/changelog.d/16684.misc b/changelog.d/16684.misc deleted file mode 100644 index 6fb55c08a57b..000000000000 --- a/changelog.d/16684.misc +++ /dev/null @@ -1 +0,0 @@ -Reoranganise test files. diff --git a/changelog.d/16695.doc b/changelog.d/16695.doc deleted file mode 100644 index 6cb284c50114..000000000000 --- a/changelog.d/16695.doc +++ /dev/null @@ -1 +0,0 @@ -Fix poetry version typo in [contributors' guide](https://matrix-org.github.io/synapse/latest/development/contributing_guide.html). diff --git a/changelog.d/16697.misc b/changelog.d/16697.misc deleted file mode 100644 index 02fd236ab43e..000000000000 --- a/changelog.d/16697.misc +++ /dev/null @@ -1 +0,0 @@ -Remove old full schema dumps which are no longer used. diff --git a/changelog.d/16699.feature b/changelog.d/16699.feature deleted file mode 100644 index 7ede50f3264c..000000000000 --- a/changelog.d/16699.feature +++ /dev/null @@ -1 +0,0 @@ -Add an autojoin setting for the notices room so users get joined directly instead of receiving an invite. diff --git a/changelog.d/16700.doc b/changelog.d/16700.doc deleted file mode 100644 index 0d55ad338a62..000000000000 --- a/changelog.d/16700.doc +++ /dev/null @@ -1 +0,0 @@ -Switch the example UNIX socket paths to /run. Add HAProxy example configuration for UNIX sockets. diff --git a/changelog.d/16701.feature b/changelog.d/16701.feature deleted file mode 100644 index 2a66fc932adc..000000000000 --- a/changelog.d/16701.feature +++ /dev/null @@ -1 +0,0 @@ -Follow redirects when downloading media over federation (per [MSC3860](https://github.com/matrix-org/matrix-spec-proposals/pull/3860)). diff --git a/changelog.d/16702.misc b/changelog.d/16702.misc deleted file mode 100644 index d97d0a11be76..000000000000 --- a/changelog.d/16702.misc +++ /dev/null @@ -1 +0,0 @@ -Raise poetry-core upper bound to <=1.8.1. This allows contributors to import Synapse after `poetry install`ing with Poetry 1.6 and above. Contributed by Mo Balaa. diff --git a/changelog.d/16704.misc b/changelog.d/16704.misc deleted file mode 100644 index 4dafb27fd8c1..000000000000 --- a/changelog.d/16704.misc +++ /dev/null @@ -1 +0,0 @@ -Add a workflow to try and automatically fixup linting in a PR. diff --git a/changelog.d/16707.feature b/changelog.d/16707.feature deleted file mode 100644 index 707502d344d5..000000000000 --- a/changelog.d/16707.feature +++ /dev/null @@ -1 +0,0 @@ -Synapse now declares support for Matrix v1.7, v1.8, and v1.9. diff --git a/changelog.d/16714.doc b/changelog.d/16714.doc deleted file mode 100644 index 511d2b60224d..000000000000 --- a/changelog.d/16714.doc +++ /dev/null @@ -1 +0,0 @@ -Add documentation for how to validate the configuration file with synapse.config script. diff --git a/changelog.d/16725.bugfix b/changelog.d/16725.bugfix deleted file mode 100644 index 42dc249fb9df..000000000000 --- a/changelog.d/16725.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a bug introduced in Synapse 1.7.2 where rooms whose power levels lacked an `events` field could not be upgraded. diff --git a/changelog.d/16729.bugfix b/changelog.d/16729.bugfix deleted file mode 100644 index 50e72ac1f817..000000000000 --- a/changelog.d/16729.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix GET /_synapse/admin/v1/federation/destinations returning null (instead of 0) for `retry_last_ts` and `retry_interval`. diff --git a/changelog.d/16730.misc b/changelog.d/16730.misc deleted file mode 100644 index baee042f2f2c..000000000000 --- a/changelog.d/16730.misc +++ /dev/null @@ -1 +0,0 @@ -Add a Postgres `REPLICA IDENTITY` to tables that do not have an implicit one. This should allow use of Postgres logical replication. \ No newline at end of file diff --git a/debian/changelog b/debian/changelog index 5ec877160480..b4626f741e04 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.98.0~rc1) stable; urgency=medium + + * New Synapse release 1.98.0rc1. + + -- Synapse Packaging team Tue, 05 Dec 2023 13:08:42 +0000 + matrix-synapse-py3 (1.97.0) stable; urgency=medium * New Synapse release 1.97.0. diff --git a/pyproject.toml b/pyproject.toml index eeede837d093..a3eb6111e023 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -96,7 +96,7 @@ module-name = "synapse.synapse_rust" [tool.poetry] name = "matrix-synapse" -version = "1.97.0" +version = "1.98.0rc1" description = "Homeserver for the Matrix decentralised comms protocol" authors = ["Matrix.org Team and Contributors "] license = "Apache-2.0" From 386649325aa17d694bd6c6b07d354a1cb67593b2 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Tue, 5 Dec 2023 13:16:59 +0000 Subject: [PATCH 27/29] Fixup dependency bumps syntax in changelog --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index c94736ea9a4b..b8d46b0532ed 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -61,7 +61,7 @@ but we are striving to make this as seamless as possible. * Bump prometheus-client from 0.18.0 to 0.19.0. ([\#16691](https://github.com/matrix-org/synapse/issues/16691)) * Bump pyasn1 from 0.5.0 to 0.5.1. ([\#16689](https://github.com/matrix-org/synapse/issues/16689)) * Bump pydantic from 2.4.2 to 2.5.1. ([\#16663](https://github.com/matrix-org/synapse/issues/16663)) -- Bump pyo3 (0.19.2→0.20.0), pythonize (0.19.0→0.20.0) and pyo3-log (0.8.1→0.9.0). ([\#16673](https://github.com/matrix-org/synapse/issues/16673)) +* Bump pyo3 (0.19.2→0.20.0), pythonize (0.19.0→0.20.0) and pyo3-log (0.8.1→0.9.0). ([\#16673](https://github.com/matrix-org/synapse/issues/16673)) * Bump pyopenssl from 23.2.0 to 23.3.0. ([\#16662](https://github.com/matrix-org/synapse/issues/16662)) * Bump ruff from 0.1.4 to 0.1.6. ([\#16690](https://github.com/matrix-org/synapse/issues/16690)) * Bump sentry-sdk from 1.32.0 to 1.35.0. ([\#16666](https://github.com/matrix-org/synapse/issues/16666)) From c8a24c55a9f8c74e3cbb601ba40f91e564c425c2 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Tue, 5 Dec 2023 13:38:09 +0000 Subject: [PATCH 28/29] Amend changelog typo --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index b8d46b0532ed..74348cf17a8b 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -31,7 +31,7 @@ but we are striving to make this as seamless as possible. ### Improved Documentation - Add schema rollback information to documentation. ([\#16661](https://github.com/matrix-org/synapse/issues/16661)) -- Fix poetry version typo in [contributors' guide](https://matrix-org.github.io/synapse/latest/development/contributing_guide.html). ([\#16695](https://github.com/matrix-org/synapse/issues/16695)) +- Fix poetry version typo in the [contributors' guide](https://matrix-org.github.io/synapse/latest/development/contributing_guide.html). ([\#16695](https://github.com/matrix-org/synapse/issues/16695)) - Switch the example UNIX socket paths to `/run`. Add HAProxy example configuration for UNIX sockets. ([\#16700](https://github.com/matrix-org/synapse/issues/16700)) - Add documentation for how to validate the configuration file with `synapse.config` script. ([\#16714](https://github.com/matrix-org/synapse/issues/16714)) From 44377f5ac05ff4e95f440f5704805b87a37e3327 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Tue, 5 Dec 2023 16:07:42 +0000 Subject: [PATCH 29/29] Revert postgres logical replication deltaas This reverts two commits: 0bb8e418a41c6f583ca9d705b400e37e2308a534 "Fix postgres schema after dropping old tables (#16730)" and 51e4e35653f98c3f61222fbdbdb1dcb8864f7fca "Add a Postgres `REPLICA IDENTITY` to tables that do not have an implicit one. This should allow use of Postgres logical replication. (take 2, now with no added deadlocks!) (#16658)" and also amends the changelog. --- CHANGES.md | 1 - ...entity_applied_module_schemas.sql.postgres | 1 - ...dentity_applied_schema_deltas.sql.postgres | 1 - ...a_identity_background_updates.sql.postgres | 1 - ...dentity_schema_compat_version.sql.postgres | 1 - ...plica_identity_schema_version.sql.postgres | 1 - .../delta/83/10_replica_identity_README.md | 13 --- ...replica_identity_account_data.sql.postgres | 1 - ...ity_application_services_txns.sql.postgres | 1 - ...identity_appservice_room_list.sql.postgres | 1 - ...ty_appservice_stream_position.sql.postgres | 1 - ...eplica_identity_blocked_rooms.sql.postgres | 1 - ...validation_stream_by_instance.sql.postgres | 1 - ...ty_current_state_delta_stream.sql.postgres | 1 - ...identity_current_state_events.sql.postgres | 1 - ...lica_identity_deleted_pushers.sql.postgres | 1 - ...dentity_device_auth_providers.sql.postgres | 1 - ...ntity_device_federation_inbox.sql.postgres | 1 - ...tity_device_federation_outbox.sql.postgres | 1 - ...replica_identity_device_inbox.sql.postgres | 1 - ...ges_converted_stream_position.sql.postgres | 1 - ..._device_lists_changes_in_room.sql.postgres | 1 - ...e_lists_outbound_last_success.sql.postgres | 1 - ...y_device_lists_outbound_pokes.sql.postgres | 1 - ...ity_device_lists_remote_cache.sql.postgres | 1 - ...vice_lists_remote_extremeties.sql.postgres | 1 - ...ty_device_lists_remote_resync.sql.postgres | 1 - ..._identity_device_lists_stream.sql.postgres | 1 - .../10_replica_identity_devices.sql.postgres | 1 - ...entity_e2e_cross_signing_keys.sql.postgres | 1 - ..._e2e_cross_signing_signatures.sql.postgres | 1 - ...identity_e2e_device_keys_json.sql.postgres | 1 - ...entity_e2e_fallback_keys_json.sql.postgres | 1 - ...entity_e2e_one_time_keys_json.sql.postgres | 1 - ...eplica_identity_e2e_room_keys.sql.postgres | 1 - ...entity_e2e_room_keys_versions.sql.postgres | 1 - ...replica_identity_erased_users.sql.postgres | 1 - ...0_replica_identity_event_auth.sql.postgres | 1 - ...entity_event_auth_chain_links.sql.postgres | 1 - ...ty_event_backward_extremities.sql.postgres | 1 - ..._replica_identity_event_edges.sql.postgres | 1 - ...ity_event_forward_extremities.sql.postgres | 1 - ...0_replica_identity_event_json.sql.postgres | 1 - ...a_identity_event_push_actions.sql.postgres | 1 - ...ty_event_push_actions_staging.sql.postgres | 1 - ...a_identity_event_push_summary.sql.postgres | 1 - ...ummary_last_receipt_stream_id.sql.postgres | 1 - ..._push_summary_stream_ordering.sql.postgres | 1 - ...lica_identity_event_relations.sql.postgres | 1 - ...replica_identity_event_search.sql.postgres | 1 - ...dentity_event_to_state_groups.sql.postgres | 1 - ...entity_event_txn_id_device_id.sql.postgres | 1 - .../10_replica_identity_events.sql.postgres | 1 - ...ration_inbound_events_staging.sql.postgres | 1 - ...ty_federation_stream_position.sql.postgres | 1 - ...eplica_identity_ignored_users.sql.postgres | 1 - ...tity_local_current_membership.sql.postgres | 1 - ...entity_local_media_repository.sql.postgres | 1 - ...l_media_repository_thumbnails.sql.postgres | 1 - ...al_media_repository_url_cache.sql.postgres | 1 - ...identity_monthly_active_users.sql.postgres | 1 - ...identity_partial_state_events.sql.postgres | 1 - ...y_partial_state_rooms_servers.sql.postgres | 1 - ...lica_identity_presence_stream.sql.postgres | 1 - .../10_replica_identity_profiles.sql.postgres | 1 - ...ca_identity_push_rules_stream.sql.postgres | 1 - ...a_identity_ratelimit_override.sql.postgres | 1 - ...plica_identity_receipts_graph.sql.postgres | 1 - ..._identity_receipts_linearized.sql.postgres | 1 - ...dentity_received_transactions.sql.postgres | 1 - ...0_replica_identity_redactions.sql.postgres | 1 - ..._identity_registration_tokens.sql.postgres | 1 - ...0_replica_identity_rejections.sql.postgres | 1 - ...a_identity_remote_media_cache.sql.postgres | 1 - ...remote_media_cache_thumbnails.sql.postgres | 1 - ...ca_identity_room_account_data.sql.postgres | 1 - ...a_identity_room_alias_servers.sql.postgres | 1 - ...replica_identity_room_aliases.sql.postgres | 1 - ...0_replica_identity_room_depth.sql.postgres | 1 - ...ity_room_forgetter_stream_pos.sql.postgres | 1 - ...ica_identity_room_memberships.sql.postgres | 1 - ...ity_room_stats_earliest_token.sql.postgres | 1 - ...ica_identity_room_stats_state.sql.postgres | 1 - ...10_replica_identity_room_tags.sql.postgres | 1 - ..._identity_room_tags_revisions.sql.postgres | 1 - ...ica_identity_server_keys_json.sql.postgres | 1 - ...dentity_server_signature_keys.sql.postgres | 1 - .../10_replica_identity_sessions.sql.postgres | 1 - ...replica_identity_state_events.sql.postgres | 1 - ...ty_stats_incremental_position.sql.postgres | 1 - ...ity_stream_ordering_to_exterm.sql.postgres | 1 - ...ica_identity_stream_positions.sql.postgres | 1 - .../10_replica_identity_threads.sql.postgres | 1 - ..._threepid_guest_access_tokens.sql.postgres | 1 - ...eplica_identity_timeline_gaps.sql.postgres | 1 - ...ica_identity_ui_auth_sessions.sql.postgres | 1 - ..._ui_auth_sessions_credentials.sql.postgres | 1 - ...identity_ui_auth_sessions_ips.sql.postgres | 1 - ...ca_identity_user_daily_visits.sql.postgres | 1 - ...plica_identity_user_directory.sql.postgres | 1 - ...dentity_user_directory_search.sql.postgres | 1 - ...ity_user_directory_stream_pos.sql.postgres | 1 - ...ca_identity_user_external_ids.sql.postgres | 1 - ...replica_identity_user_filters.sql.postgres | 1 - .../10_replica_identity_user_ips.sql.postgres | 1 - ...dentity_user_signature_stream.sql.postgres | 1 - ...ntity_user_threepid_id_server.sql.postgres | 1 - ...plica_identity_user_threepids.sql.postgres | 1 - .../83/10_replica_identity_users.sql.postgres | 1 - ...dentity_users_in_public_rooms.sql.postgres | 1 - ...ty_users_pending_deactivation.sql.postgres | 1 - ...users_who_share_private_rooms.sql.postgres | 1 - ...replica_identity_worker_locks.sql.postgres | 1 - ...ntity_worker_read_write_locks.sql.postgres | 1 - ..._worker_read_write_locks_mode.sql.postgres | 1 - ...ca_identity_state_group_edges.sql.postgres | 1 - ...a_identity_state_groups_state.sql.postgres | 1 - tests/storage/test_database.py | 85 +------------------ 118 files changed, 1 insertion(+), 213 deletions(-) delete mode 100644 synapse/storage/schema/common/delta/83/10_replica_identity_applied_module_schemas.sql.postgres delete mode 100644 synapse/storage/schema/common/delta/83/10_replica_identity_applied_schema_deltas.sql.postgres delete mode 100644 synapse/storage/schema/common/delta/83/10_replica_identity_background_updates.sql.postgres delete mode 100644 synapse/storage/schema/common/delta/83/10_replica_identity_schema_compat_version.sql.postgres delete mode 100644 synapse/storage/schema/common/delta/83/10_replica_identity_schema_version.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_README.md delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_account_data.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_application_services_txns.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_appservice_room_list.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_appservice_stream_position.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_blocked_rooms.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_cache_invalidation_stream_by_instance.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_current_state_delta_stream.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_current_state_events.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_deleted_pushers.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_device_auth_providers.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_device_federation_inbox.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_device_federation_outbox.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_device_inbox.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_device_lists_changes_converted_stream_position.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_device_lists_changes_in_room.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_device_lists_outbound_last_success.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_device_lists_outbound_pokes.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_device_lists_remote_cache.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_device_lists_remote_extremeties.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_device_lists_remote_resync.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_device_lists_stream.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_devices.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_e2e_cross_signing_keys.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_e2e_cross_signing_signatures.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_e2e_device_keys_json.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_e2e_fallback_keys_json.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_e2e_one_time_keys_json.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_e2e_room_keys.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_e2e_room_keys_versions.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_erased_users.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_event_auth.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_event_auth_chain_links.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_event_backward_extremities.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_event_edges.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_event_forward_extremities.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_event_json.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_event_push_actions.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_event_push_actions_staging.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_event_push_summary.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_event_push_summary_last_receipt_stream_id.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_event_push_summary_stream_ordering.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_event_relations.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_event_search.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_event_to_state_groups.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_event_txn_id_device_id.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_events.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_federation_inbound_events_staging.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_federation_stream_position.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_ignored_users.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_local_current_membership.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_local_media_repository.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_local_media_repository_thumbnails.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_local_media_repository_url_cache.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_monthly_active_users.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_partial_state_events.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_partial_state_rooms_servers.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_presence_stream.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_profiles.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_push_rules_stream.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_ratelimit_override.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_receipts_graph.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_receipts_linearized.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_received_transactions.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_redactions.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_registration_tokens.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_rejections.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_remote_media_cache.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_remote_media_cache_thumbnails.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_room_account_data.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_room_alias_servers.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_room_aliases.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_room_depth.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_room_forgetter_stream_pos.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_room_memberships.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_room_stats_earliest_token.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_room_stats_state.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_room_tags.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_room_tags_revisions.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_server_keys_json.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_server_signature_keys.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_sessions.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_state_events.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_stats_incremental_position.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_stream_ordering_to_exterm.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_stream_positions.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_threads.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_threepid_guest_access_tokens.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_timeline_gaps.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_ui_auth_sessions.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_ui_auth_sessions_credentials.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_ui_auth_sessions_ips.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_user_daily_visits.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_user_directory.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_user_directory_search.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_user_directory_stream_pos.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_user_external_ids.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_user_filters.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_user_ips.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_user_signature_stream.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_user_threepid_id_server.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_user_threepids.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_users.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_users_in_public_rooms.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_users_pending_deactivation.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_users_who_share_private_rooms.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_worker_locks.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_worker_read_write_locks.sql.postgres delete mode 100644 synapse/storage/schema/main/delta/83/10_replica_identity_worker_read_write_locks_mode.sql.postgres delete mode 100644 synapse/storage/schema/state/delta/83/10_replica_identity_state_group_edges.sql.postgres delete mode 100644 synapse/storage/schema/state/delta/83/10_replica_identity_state_groups_state.sql.postgres diff --git a/CHANGES.md b/CHANGES.md index 74348cf17a8b..a52173609da2 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -39,7 +39,6 @@ but we are striving to make this as seamless as possible. - Clean-up unused tables. ([\#16522](https://github.com/matrix-org/synapse/issues/16522)) - Reduce a little database load while processing state auth chains. ([\#16552](https://github.com/matrix-org/synapse/issues/16552)) -- Add a Postgres `REPLICA IDENTITY` to tables that do not have an implicit one. This should allow use of Postgres logical replication. ([\#16658](https://github.com/matrix-org/synapse/issues/16658), [\#16730](https://github.com/matrix-org/synapse/issues/16730)) - Reduce database load of pruning old `user_ips`. ([\#16667](https://github.com/matrix-org/synapse/issues/16667)) - Reduce DB load when forget on leave setting is disabled. ([\#16668](https://github.com/matrix-org/synapse/issues/16668)) - Ignore `encryption_enabled_by_default_for_room_type` setting when creating server notices room, since the notices will be send unencrypted anyway. ([\#16677](https://github.com/matrix-org/synapse/issues/16677)) diff --git a/synapse/storage/schema/common/delta/83/10_replica_identity_applied_module_schemas.sql.postgres b/synapse/storage/schema/common/delta/83/10_replica_identity_applied_module_schemas.sql.postgres deleted file mode 100644 index 57c35b64301a..000000000000 --- a/synapse/storage/schema/common/delta/83/10_replica_identity_applied_module_schemas.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE applied_module_schemas REPLICA IDENTITY USING INDEX applied_module_schemas_module_name_file_key; diff --git a/synapse/storage/schema/common/delta/83/10_replica_identity_applied_schema_deltas.sql.postgres b/synapse/storage/schema/common/delta/83/10_replica_identity_applied_schema_deltas.sql.postgres deleted file mode 100644 index 1426f9a9e402..000000000000 --- a/synapse/storage/schema/common/delta/83/10_replica_identity_applied_schema_deltas.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE applied_schema_deltas REPLICA IDENTITY USING INDEX applied_schema_deltas_version_file_key; diff --git a/synapse/storage/schema/common/delta/83/10_replica_identity_background_updates.sql.postgres b/synapse/storage/schema/common/delta/83/10_replica_identity_background_updates.sql.postgres deleted file mode 100644 index f04937b3db35..000000000000 --- a/synapse/storage/schema/common/delta/83/10_replica_identity_background_updates.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE background_updates REPLICA IDENTITY USING INDEX background_updates_uniqueness; diff --git a/synapse/storage/schema/common/delta/83/10_replica_identity_schema_compat_version.sql.postgres b/synapse/storage/schema/common/delta/83/10_replica_identity_schema_compat_version.sql.postgres deleted file mode 100644 index e3a6f7ca4ffe..000000000000 --- a/synapse/storage/schema/common/delta/83/10_replica_identity_schema_compat_version.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE schema_compat_version REPLICA IDENTITY USING INDEX schema_compat_version_lock_key; diff --git a/synapse/storage/schema/common/delta/83/10_replica_identity_schema_version.sql.postgres b/synapse/storage/schema/common/delta/83/10_replica_identity_schema_version.sql.postgres deleted file mode 100644 index c10efe05637a..000000000000 --- a/synapse/storage/schema/common/delta/83/10_replica_identity_schema_version.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE schema_version REPLICA IDENTITY USING INDEX schema_version_lock_key; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_README.md b/synapse/storage/schema/main/delta/83/10_replica_identity_README.md deleted file mode 100644 index fd7bdfc7adea..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_README.md +++ /dev/null @@ -1,13 +0,0 @@ -The `10_replica_identity_xxx.sql.postgres` series of schema deltas adds replica identities for tables that do not have one implicitly as a result of having a primary key. - -This is needed to use logical replication with Synapse (at least without `UPDATE` and `DELETE` statements failing!). - -Where possible, we use an existing `UNIQUE` index on `NOT NULL` columns as the replica identity. Otherwise, we have to fall back to using the full row as a replica identity. - -Unfortunately, by running all the `ALTER TABLE` statements in one schema delta per database, it was too likely to hit a deadlock as it would only take -one other transaction from a running Synapse worker to access the tables out of order and trigger a deadlock. - -By having each statement in its own delta file, each one is run in its own transaction and only needs to take a very brief (instant) lock on the table but no other tables, -so there should be no chance of deadlock. - -Like many schema deltas we already apply to Synapse, it is probably blocked by an ongoing `pg_dump`. diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_account_data.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_account_data.sql.postgres deleted file mode 100644 index 1bf720e80ee2..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_account_data.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE account_data REPLICA IDENTITY USING INDEX account_data_uniqueness; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_application_services_txns.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_application_services_txns.sql.postgres deleted file mode 100644 index 4ec052ebebb7..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_application_services_txns.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE application_services_txns REPLICA IDENTITY USING INDEX application_services_txns_as_id_txn_id_key; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_appservice_room_list.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_appservice_room_list.sql.postgres deleted file mode 100644 index 232a628ca2ab..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_appservice_room_list.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE appservice_room_list REPLICA IDENTITY USING INDEX appservice_room_list_idx; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_appservice_stream_position.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_appservice_stream_position.sql.postgres deleted file mode 100644 index e870a68723e5..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_appservice_stream_position.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE appservice_stream_position REPLICA IDENTITY USING INDEX appservice_stream_position_lock_key; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_blocked_rooms.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_blocked_rooms.sql.postgres deleted file mode 100644 index e3fce4045ff4..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_blocked_rooms.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE blocked_rooms REPLICA IDENTITY USING INDEX blocked_rooms_idx; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_cache_invalidation_stream_by_instance.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_cache_invalidation_stream_by_instance.sql.postgres deleted file mode 100644 index 4fd87faaf62d..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_cache_invalidation_stream_by_instance.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE cache_invalidation_stream_by_instance REPLICA IDENTITY USING INDEX cache_invalidation_stream_by_instance_id; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_current_state_delta_stream.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_current_state_delta_stream.sql.postgres deleted file mode 100644 index b618286e8a99..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_current_state_delta_stream.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE current_state_delta_stream REPLICA IDENTITY FULL; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_current_state_events.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_current_state_events.sql.postgres deleted file mode 100644 index b058a9f06e0d..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_current_state_events.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE current_state_events REPLICA IDENTITY USING INDEX current_state_events_event_id_key; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_deleted_pushers.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_deleted_pushers.sql.postgres deleted file mode 100644 index 0b4fcc5ae158..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_deleted_pushers.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE deleted_pushers REPLICA IDENTITY FULL; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_device_auth_providers.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_device_auth_providers.sql.postgres deleted file mode 100644 index 8e02438b476a..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_device_auth_providers.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE device_auth_providers REPLICA IDENTITY FULL; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_device_federation_inbox.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_device_federation_inbox.sql.postgres deleted file mode 100644 index 547e485f0071..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_device_federation_inbox.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE device_federation_inbox REPLICA IDENTITY FULL; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_device_federation_outbox.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_device_federation_outbox.sql.postgres deleted file mode 100644 index 809afe4dc5ff..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_device_federation_outbox.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE device_federation_outbox REPLICA IDENTITY FULL; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_device_inbox.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_device_inbox.sql.postgres deleted file mode 100644 index 07652038a262..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_device_inbox.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE device_inbox REPLICA IDENTITY FULL; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_device_lists_changes_converted_stream_position.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_device_lists_changes_converted_stream_position.sql.postgres deleted file mode 100644 index 8474b8c84288..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_device_lists_changes_converted_stream_position.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE device_lists_changes_converted_stream_position REPLICA IDENTITY USING INDEX device_lists_changes_converted_stream_position_lock_key; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_device_lists_changes_in_room.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_device_lists_changes_in_room.sql.postgres deleted file mode 100644 index 712a54bc0f5c..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_device_lists_changes_in_room.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE device_lists_changes_in_room REPLICA IDENTITY USING INDEX device_lists_changes_in_stream_id; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_device_lists_outbound_last_success.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_device_lists_outbound_last_success.sql.postgres deleted file mode 100644 index 4357690e8de3..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_device_lists_outbound_last_success.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE device_lists_outbound_last_success REPLICA IDENTITY USING INDEX device_lists_outbound_last_success_unique_idx; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_device_lists_outbound_pokes.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_device_lists_outbound_pokes.sql.postgres deleted file mode 100644 index cc724877cffc..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_device_lists_outbound_pokes.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE device_lists_outbound_pokes REPLICA IDENTITY FULL; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_device_lists_remote_cache.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_device_lists_remote_cache.sql.postgres deleted file mode 100644 index ea4351c65473..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_device_lists_remote_cache.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE device_lists_remote_cache REPLICA IDENTITY USING INDEX device_lists_remote_cache_unique_id; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_device_lists_remote_extremeties.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_device_lists_remote_extremeties.sql.postgres deleted file mode 100644 index 9dc3bec6fa57..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_device_lists_remote_extremeties.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE device_lists_remote_extremeties REPLICA IDENTITY USING INDEX device_lists_remote_extremeties_unique_idx; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_device_lists_remote_resync.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_device_lists_remote_resync.sql.postgres deleted file mode 100644 index df3cecfb9439..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_device_lists_remote_resync.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE device_lists_remote_resync REPLICA IDENTITY USING INDEX device_lists_remote_resync_idx; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_device_lists_stream.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_device_lists_stream.sql.postgres deleted file mode 100644 index 30a79226139e..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_device_lists_stream.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE device_lists_stream REPLICA IDENTITY FULL; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_devices.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_devices.sql.postgres deleted file mode 100644 index 57d54a0dc0b5..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_devices.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE devices REPLICA IDENTITY USING INDEX device_uniqueness; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_e2e_cross_signing_keys.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_e2e_cross_signing_keys.sql.postgres deleted file mode 100644 index f3f26936313f..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_e2e_cross_signing_keys.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE e2e_cross_signing_keys REPLICA IDENTITY USING INDEX e2e_cross_signing_keys_stream_idx; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_e2e_cross_signing_signatures.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_e2e_cross_signing_signatures.sql.postgres deleted file mode 100644 index d2347b71538e..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_e2e_cross_signing_signatures.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE e2e_cross_signing_signatures REPLICA IDENTITY FULL; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_e2e_device_keys_json.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_e2e_device_keys_json.sql.postgres deleted file mode 100644 index 66dafdc8e1e7..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_e2e_device_keys_json.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE e2e_device_keys_json REPLICA IDENTITY USING INDEX e2e_device_keys_json_uniqueness; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_e2e_fallback_keys_json.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_e2e_fallback_keys_json.sql.postgres deleted file mode 100644 index 8bb4dfc78b5d..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_e2e_fallback_keys_json.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE e2e_fallback_keys_json REPLICA IDENTITY USING INDEX e2e_fallback_keys_json_uniqueness; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_e2e_one_time_keys_json.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_e2e_one_time_keys_json.sql.postgres deleted file mode 100644 index b6609cfdfbc1..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_e2e_one_time_keys_json.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE e2e_one_time_keys_json REPLICA IDENTITY USING INDEX e2e_one_time_keys_json_uniqueness; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_e2e_room_keys.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_e2e_room_keys.sql.postgres deleted file mode 100644 index 9b98924165c3..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_e2e_room_keys.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE e2e_room_keys REPLICA IDENTITY USING INDEX e2e_room_keys_with_version_idx; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_e2e_room_keys_versions.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_e2e_room_keys_versions.sql.postgres deleted file mode 100644 index ac7aaba44de9..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_e2e_room_keys_versions.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE e2e_room_keys_versions REPLICA IDENTITY USING INDEX e2e_room_keys_versions_idx; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_erased_users.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_erased_users.sql.postgres deleted file mode 100644 index c775f81333cd..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_erased_users.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE erased_users REPLICA IDENTITY USING INDEX erased_users_user; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_event_auth.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_event_auth.sql.postgres deleted file mode 100644 index d8014d123bc8..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_event_auth.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE event_auth REPLICA IDENTITY FULL; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_event_auth_chain_links.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_event_auth_chain_links.sql.postgres deleted file mode 100644 index c2f9c90d42e0..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_event_auth_chain_links.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE event_auth_chain_links REPLICA IDENTITY FULL; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_event_backward_extremities.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_event_backward_extremities.sql.postgres deleted file mode 100644 index 751541a929a9..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_event_backward_extremities.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE event_backward_extremities REPLICA IDENTITY USING INDEX event_backward_extremities_event_id_room_id_key; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_event_edges.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_event_edges.sql.postgres deleted file mode 100644 index 066ea763f1ef..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_event_edges.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE event_edges REPLICA IDENTITY USING INDEX event_edges_event_id_prev_event_id_idx; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_event_forward_extremities.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_event_forward_extremities.sql.postgres deleted file mode 100644 index d70d6fd55c57..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_event_forward_extremities.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE event_forward_extremities REPLICA IDENTITY USING INDEX event_forward_extremities_event_id_room_id_key; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_event_json.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_event_json.sql.postgres deleted file mode 100644 index b614b9642292..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_event_json.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE event_json REPLICA IDENTITY USING INDEX event_json_event_id_key; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_event_push_actions.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_event_push_actions.sql.postgres deleted file mode 100644 index 7a92620270a8..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_event_push_actions.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE event_push_actions REPLICA IDENTITY FULL; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_event_push_actions_staging.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_event_push_actions_staging.sql.postgres deleted file mode 100644 index 292ed9787d8f..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_event_push_actions_staging.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE event_push_actions_staging REPLICA IDENTITY FULL; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_event_push_summary.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_event_push_summary.sql.postgres deleted file mode 100644 index 42a9b6b6762a..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_event_push_summary.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE event_push_summary REPLICA IDENTITY FULL; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_event_push_summary_last_receipt_stream_id.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_event_push_summary_last_receipt_stream_id.sql.postgres deleted file mode 100644 index 707e498a6ebf..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_event_push_summary_last_receipt_stream_id.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE event_push_summary_last_receipt_stream_id REPLICA IDENTITY USING INDEX event_push_summary_last_receipt_stream_id_lock_key; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_event_push_summary_stream_ordering.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_event_push_summary_stream_ordering.sql.postgres deleted file mode 100644 index 1240e3e1b51d..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_event_push_summary_stream_ordering.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE event_push_summary_stream_ordering REPLICA IDENTITY USING INDEX event_push_summary_stream_ordering_lock_key; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_event_relations.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_event_relations.sql.postgres deleted file mode 100644 index 6b62a6bc2003..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_event_relations.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE event_relations REPLICA IDENTITY USING INDEX event_relations_id; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_event_search.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_event_search.sql.postgres deleted file mode 100644 index e8467fc3d37c..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_event_search.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE event_search REPLICA IDENTITY FULL; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_event_to_state_groups.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_event_to_state_groups.sql.postgres deleted file mode 100644 index 4206975c3563..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_event_to_state_groups.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE event_to_state_groups REPLICA IDENTITY USING INDEX event_to_state_groups_event_id_key; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_event_txn_id_device_id.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_event_txn_id_device_id.sql.postgres deleted file mode 100644 index 05fce1e6b094..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_event_txn_id_device_id.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE event_txn_id_device_id REPLICA IDENTITY USING INDEX event_txn_id_device_id_event_id; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_events.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_events.sql.postgres deleted file mode 100644 index 9046b8eb08b5..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_events.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE events REPLICA IDENTITY USING INDEX events_event_id_key; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_federation_inbound_events_staging.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_federation_inbound_events_staging.sql.postgres deleted file mode 100644 index d9d2f88f675a..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_federation_inbound_events_staging.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE federation_inbound_events_staging REPLICA IDENTITY USING INDEX federation_inbound_events_staging_instance_event; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_federation_stream_position.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_federation_stream_position.sql.postgres deleted file mode 100644 index 5f9df14d7dcf..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_federation_stream_position.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE federation_stream_position REPLICA IDENTITY USING INDEX federation_stream_position_instance; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_ignored_users.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_ignored_users.sql.postgres deleted file mode 100644 index dd082b7f9e3e..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_ignored_users.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE ignored_users REPLICA IDENTITY USING INDEX ignored_users_uniqueness; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_local_current_membership.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_local_current_membership.sql.postgres deleted file mode 100644 index f36bad701a5d..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_local_current_membership.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE local_current_membership REPLICA IDENTITY USING INDEX local_current_membership_idx; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_local_media_repository.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_local_media_repository.sql.postgres deleted file mode 100644 index 78a5128af384..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_local_media_repository.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE local_media_repository REPLICA IDENTITY FULL; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_local_media_repository_thumbnails.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_local_media_repository_thumbnails.sql.postgres deleted file mode 100644 index 6cd53f8e15aa..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_local_media_repository_thumbnails.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE local_media_repository_thumbnails REPLICA IDENTITY FULL; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_local_media_repository_url_cache.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_local_media_repository_url_cache.sql.postgres deleted file mode 100644 index a41590ce6bee..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_local_media_repository_url_cache.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE local_media_repository_url_cache REPLICA IDENTITY FULL; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_monthly_active_users.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_monthly_active_users.sql.postgres deleted file mode 100644 index 326fc279d3c0..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_monthly_active_users.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE monthly_active_users REPLICA IDENTITY USING INDEX monthly_active_users_users; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_partial_state_events.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_partial_state_events.sql.postgres deleted file mode 100644 index c6acd84315be..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_partial_state_events.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE partial_state_events REPLICA IDENTITY USING INDEX partial_state_events_event_id_key; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_partial_state_rooms_servers.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_partial_state_rooms_servers.sql.postgres deleted file mode 100644 index c618660b5f50..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_partial_state_rooms_servers.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE partial_state_rooms_servers REPLICA IDENTITY USING INDEX partial_state_rooms_servers_room_id_server_name_key; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_presence_stream.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_presence_stream.sql.postgres deleted file mode 100644 index 4e5d8d7a68d4..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_presence_stream.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE presence_stream REPLICA IDENTITY FULL; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_profiles.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_profiles.sql.postgres deleted file mode 100644 index bb25e7a67a12..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_profiles.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE profiles REPLICA IDENTITY USING INDEX profiles_user_id_key; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_push_rules_stream.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_push_rules_stream.sql.postgres deleted file mode 100644 index f305d9a73a52..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_push_rules_stream.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE push_rules_stream REPLICA IDENTITY FULL; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_ratelimit_override.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_ratelimit_override.sql.postgres deleted file mode 100644 index f10f359155bf..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_ratelimit_override.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE ratelimit_override REPLICA IDENTITY USING INDEX ratelimit_override_idx; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_receipts_graph.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_receipts_graph.sql.postgres deleted file mode 100644 index 954dd9266771..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_receipts_graph.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE receipts_graph REPLICA IDENTITY FULL; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_receipts_linearized.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_receipts_linearized.sql.postgres deleted file mode 100644 index 302d88753189..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_receipts_linearized.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE receipts_linearized REPLICA IDENTITY FULL; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_received_transactions.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_received_transactions.sql.postgres deleted file mode 100644 index 3b8c2277172e..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_received_transactions.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE received_transactions REPLICA IDENTITY FULL; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_redactions.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_redactions.sql.postgres deleted file mode 100644 index ffa3444b955b..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_redactions.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE redactions REPLICA IDENTITY USING INDEX redactions_event_id_key; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_registration_tokens.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_registration_tokens.sql.postgres deleted file mode 100644 index 3fdb5bbd54a1..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_registration_tokens.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE registration_tokens REPLICA IDENTITY USING INDEX registration_tokens_token_key; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_rejections.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_rejections.sql.postgres deleted file mode 100644 index c7a7218578d8..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_rejections.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE rejections REPLICA IDENTITY USING INDEX rejections_event_id_key; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_remote_media_cache.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_remote_media_cache.sql.postgres deleted file mode 100644 index d0ecfe1a759b..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_remote_media_cache.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE remote_media_cache REPLICA IDENTITY FULL; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_remote_media_cache_thumbnails.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_remote_media_cache_thumbnails.sql.postgres deleted file mode 100644 index 9d7b63dfb42a..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_remote_media_cache_thumbnails.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE remote_media_cache_thumbnails REPLICA IDENTITY FULL; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_room_account_data.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_room_account_data.sql.postgres deleted file mode 100644 index 36e756f6b198..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_room_account_data.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE room_account_data REPLICA IDENTITY USING INDEX room_account_data_uniqueness; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_room_alias_servers.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_room_alias_servers.sql.postgres deleted file mode 100644 index df550076ac5c..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_room_alias_servers.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE room_alias_servers REPLICA IDENTITY FULL; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_room_aliases.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_room_aliases.sql.postgres deleted file mode 100644 index bfe8db5375d8..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_room_aliases.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE room_aliases REPLICA IDENTITY USING INDEX room_aliases_room_alias_key; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_room_depth.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_room_depth.sql.postgres deleted file mode 100644 index 77c78ee4ab49..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_room_depth.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE room_depth REPLICA IDENTITY USING INDEX room_depth_room_id_key; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_room_forgetter_stream_pos.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_room_forgetter_stream_pos.sql.postgres deleted file mode 100644 index 51de67b54947..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_room_forgetter_stream_pos.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE room_forgetter_stream_pos REPLICA IDENTITY USING INDEX room_forgetter_stream_pos_lock_key; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_room_memberships.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_room_memberships.sql.postgres deleted file mode 100644 index 5819409058d1..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_room_memberships.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE room_memberships REPLICA IDENTITY USING INDEX room_memberships_event_id_key; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_room_stats_earliest_token.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_room_stats_earliest_token.sql.postgres deleted file mode 100644 index bacd9587d28f..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_room_stats_earliest_token.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE room_stats_earliest_token REPLICA IDENTITY USING INDEX room_stats_earliest_token_idx; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_room_stats_state.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_room_stats_state.sql.postgres deleted file mode 100644 index a64db05f4471..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_room_stats_state.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE room_stats_state REPLICA IDENTITY USING INDEX room_stats_state_room; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_room_tags.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_room_tags.sql.postgres deleted file mode 100644 index cc6970475189..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_room_tags.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE room_tags REPLICA IDENTITY USING INDEX room_tag_uniqueness; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_room_tags_revisions.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_room_tags_revisions.sql.postgres deleted file mode 100644 index d31063277c9e..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_room_tags_revisions.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE room_tags_revisions REPLICA IDENTITY USING INDEX room_tag_revisions_uniqueness; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_server_keys_json.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_server_keys_json.sql.postgres deleted file mode 100644 index 9c0bfb5623c2..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_server_keys_json.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE server_keys_json REPLICA IDENTITY USING INDEX server_keys_json_uniqueness; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_server_signature_keys.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_server_signature_keys.sql.postgres deleted file mode 100644 index 1b3d5e68e5b6..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_server_signature_keys.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE server_signature_keys REPLICA IDENTITY FULL; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_sessions.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_sessions.sql.postgres deleted file mode 100644 index eb8d621795b0..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_sessions.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE sessions REPLICA IDENTITY USING INDEX sessions_session_type_session_id_key; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_state_events.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_state_events.sql.postgres deleted file mode 100644 index 7e23c8a0545a..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_state_events.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE state_events REPLICA IDENTITY USING INDEX state_events_event_id_key; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_stats_incremental_position.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_stats_incremental_position.sql.postgres deleted file mode 100644 index a4a700a3a408..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_stats_incremental_position.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE stats_incremental_position REPLICA IDENTITY USING INDEX stats_incremental_position_lock_key; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_stream_ordering_to_exterm.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_stream_ordering_to_exterm.sql.postgres deleted file mode 100644 index a70074189610..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_stream_ordering_to_exterm.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE stream_ordering_to_exterm REPLICA IDENTITY FULL; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_stream_positions.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_stream_positions.sql.postgres deleted file mode 100644 index 83ff1c382542..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_stream_positions.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE stream_positions REPLICA IDENTITY USING INDEX stream_positions_idx; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_threads.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_threads.sql.postgres deleted file mode 100644 index a3891e4208e8..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_threads.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE threads REPLICA IDENTITY USING INDEX threads_uniqueness; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_threepid_guest_access_tokens.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_threepid_guest_access_tokens.sql.postgres deleted file mode 100644 index df31cf4f1d6c..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_threepid_guest_access_tokens.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE threepid_guest_access_tokens REPLICA IDENTITY FULL; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_timeline_gaps.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_timeline_gaps.sql.postgres deleted file mode 100644 index 8fb41226c14c..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_timeline_gaps.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE timeline_gaps REPLICA IDENTITY FULL; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_ui_auth_sessions.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_ui_auth_sessions.sql.postgres deleted file mode 100644 index 5d2af97f729b..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_ui_auth_sessions.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE ui_auth_sessions REPLICA IDENTITY USING INDEX ui_auth_sessions_session_id_key; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_ui_auth_sessions_credentials.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_ui_auth_sessions_credentials.sql.postgres deleted file mode 100644 index efb2359faa7e..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_ui_auth_sessions_credentials.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE ui_auth_sessions_credentials REPLICA IDENTITY USING INDEX ui_auth_sessions_credentials_session_id_stage_type_key; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_ui_auth_sessions_ips.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_ui_auth_sessions_ips.sql.postgres deleted file mode 100644 index a4b86eaf0719..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_ui_auth_sessions_ips.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE ui_auth_sessions_ips REPLICA IDENTITY USING INDEX ui_auth_sessions_ips_session_id_ip_user_agent_key; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_user_daily_visits.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_user_daily_visits.sql.postgres deleted file mode 100644 index bb22ec5335c1..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_user_daily_visits.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE user_daily_visits REPLICA IDENTITY FULL; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_user_directory.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_user_directory.sql.postgres deleted file mode 100644 index 947f8c5c0fa6..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_user_directory.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE user_directory REPLICA IDENTITY USING INDEX user_directory_user_idx; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_user_directory_search.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_user_directory_search.sql.postgres deleted file mode 100644 index d4217799440a..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_user_directory_search.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE user_directory_search REPLICA IDENTITY USING INDEX user_directory_search_user_idx; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_user_directory_stream_pos.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_user_directory_stream_pos.sql.postgres deleted file mode 100644 index 59201750b0e5..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_user_directory_stream_pos.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE user_directory_stream_pos REPLICA IDENTITY USING INDEX user_directory_stream_pos_lock_key; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_user_external_ids.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_user_external_ids.sql.postgres deleted file mode 100644 index 219ee16ddea6..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_user_external_ids.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE user_external_ids REPLICA IDENTITY USING INDEX user_external_ids_auth_provider_external_id_key; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_user_filters.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_user_filters.sql.postgres deleted file mode 100644 index 5d3e0ae475be..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_user_filters.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE user_filters REPLICA IDENTITY FULL; -- sadly the `CHECK` constraint is not enough here diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_user_ips.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_user_ips.sql.postgres deleted file mode 100644 index 5a1d8770d0c3..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_user_ips.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE user_ips REPLICA IDENTITY USING INDEX user_ips_user_token_ip_unique_index; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_user_signature_stream.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_user_signature_stream.sql.postgres deleted file mode 100644 index 8bc27df00a2b..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_user_signature_stream.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE user_signature_stream REPLICA IDENTITY USING INDEX user_signature_stream_idx; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_user_threepid_id_server.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_user_threepid_id_server.sql.postgres deleted file mode 100644 index a0ba8e67731a..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_user_threepid_id_server.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE user_threepid_id_server REPLICA IDENTITY USING INDEX user_threepid_id_server_idx; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_user_threepids.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_user_threepids.sql.postgres deleted file mode 100644 index 9c70c9011e50..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_user_threepids.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE user_threepids REPLICA IDENTITY USING INDEX medium_address; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_users.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_users.sql.postgres deleted file mode 100644 index 725e6cdc7667..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_users.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE users REPLICA IDENTITY FULL; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_users_in_public_rooms.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_users_in_public_rooms.sql.postgres deleted file mode 100644 index 86988a55505f..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_users_in_public_rooms.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE users_in_public_rooms REPLICA IDENTITY USING INDEX users_in_public_rooms_u_idx; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_users_pending_deactivation.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_users_pending_deactivation.sql.postgres deleted file mode 100644 index 7e72946d14a1..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_users_pending_deactivation.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE users_pending_deactivation REPLICA IDENTITY FULL; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_users_who_share_private_rooms.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_users_who_share_private_rooms.sql.postgres deleted file mode 100644 index 6c7213920aa7..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_users_who_share_private_rooms.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE users_who_share_private_rooms REPLICA IDENTITY USING INDEX users_who_share_private_rooms_u_idx; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_worker_locks.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_worker_locks.sql.postgres deleted file mode 100644 index 1bba45e05c47..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_worker_locks.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE worker_locks REPLICA IDENTITY USING INDEX worker_locks_key; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_worker_read_write_locks.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_worker_read_write_locks.sql.postgres deleted file mode 100644 index b9d35eb4f693..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_worker_read_write_locks.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE worker_read_write_locks REPLICA IDENTITY USING INDEX worker_read_write_locks_key; diff --git a/synapse/storage/schema/main/delta/83/10_replica_identity_worker_read_write_locks_mode.sql.postgres b/synapse/storage/schema/main/delta/83/10_replica_identity_worker_read_write_locks_mode.sql.postgres deleted file mode 100644 index 5871c98deeee..000000000000 --- a/synapse/storage/schema/main/delta/83/10_replica_identity_worker_read_write_locks_mode.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE worker_read_write_locks_mode REPLICA IDENTITY USING INDEX worker_read_write_locks_mode_key; diff --git a/synapse/storage/schema/state/delta/83/10_replica_identity_state_group_edges.sql.postgres b/synapse/storage/schema/state/delta/83/10_replica_identity_state_group_edges.sql.postgres deleted file mode 100644 index aa8fc2514cb6..000000000000 --- a/synapse/storage/schema/state/delta/83/10_replica_identity_state_group_edges.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE state_group_edges REPLICA IDENTITY USING INDEX state_group_edges_unique_idx; diff --git a/synapse/storage/schema/state/delta/83/10_replica_identity_state_groups_state.sql.postgres b/synapse/storage/schema/state/delta/83/10_replica_identity_state_groups_state.sql.postgres deleted file mode 100644 index 438603197724..000000000000 --- a/synapse/storage/schema/state/delta/83/10_replica_identity_state_groups_state.sql.postgres +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE state_groups_state REPLICA IDENTITY FULL; diff --git a/tests/storage/test_database.py b/tests/storage/test_database.py index aa8c76f18777..4d0ebb550d29 100644 --- a/tests/storage/test_database.py +++ b/tests/storage/test_database.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Callable, List, Tuple +from typing import Callable, Tuple from unittest.mock import Mock, call from twisted.internet import defer @@ -29,7 +29,6 @@ from synapse.util import Clock from tests import unittest -from tests.utils import USE_POSTGRES_FOR_TESTS class TupleComparisonClauseTestCase(unittest.TestCase): @@ -280,85 +279,3 @@ def _test_txn(txn: LoggingTransaction) -> None: ] ) self.assertEqual(exception_callback.call_count, 6) # no additional calls - - -class PostgresReplicaIdentityTestCase(unittest.HomeserverTestCase): - if not USE_POSTGRES_FOR_TESTS: - skip = "Requires Postgres" - - def prepare( - self, reactor: MemoryReactor, clock: Clock, homeserver: HomeServer - ) -> None: - self.db_pools = homeserver.get_datastores().databases - - def test_all_tables_have_postgres_replica_identity(self) -> None: - """ - Tests that all tables have a Postgres REPLICA IDENTITY. - (See https://github.com/matrix-org/synapse/issues/16224). - - Tables with a PRIMARY KEY have an implied REPLICA IDENTITY and are fine. - Other tables need them to be set with `ALTER TABLE`. - - A REPLICA IDENTITY is required for Postgres logical replication to work - properly without blocking updates and deletes. - """ - - sql = """ - -- Select tables that have no primary key and use the default replica identity rule - -- (the default is to use the primary key) - WITH tables_no_pkey AS ( - SELECT tbl.table_schema, tbl.table_name - FROM information_schema.tables tbl - WHERE table_type = 'BASE TABLE' - AND table_schema not in ('pg_catalog', 'information_schema') - AND NOT EXISTS ( - SELECT 1 - FROM information_schema.table_constraints tc - WHERE tc.constraint_type = 'PRIMARY KEY' - AND tc.table_schema = tbl.table_schema - AND tc.table_name = tbl.table_name - ) - ) - SELECT pg_class.oid::regclass FROM tables_no_pkey INNER JOIN pg_class ON pg_class.oid::regclass = table_name::regclass - WHERE relreplident = 'd' - - UNION - - -- Also select tables that use an index as a replica identity - -- but where the index doesn't exist - -- (e.g. it could have been deleted) - SELECT pg_class.oid::regclass - FROM information_schema.tables tbl - INNER JOIN pg_class ON pg_class.oid::regclass = table_name::regclass - WHERE table_type = 'BASE TABLE' - AND table_schema not in ('pg_catalog', 'information_schema') - - -- 'i' means an index is used as the replica identity - AND relreplident = 'i' - - -- look for indices that are marked as the replica identity - AND NOT EXISTS ( - SELECT indexrelid::regclass - FROM pg_index - WHERE indrelid = pg_class.oid::regclass AND indisreplident - ) - """ - - def _list_tables_with_missing_replica_identities_txn( - txn: LoggingTransaction, - ) -> List[str]: - txn.execute(sql) - return [table_name for table_name, in txn] - - for pool in self.db_pools: - missing = self.get_success( - pool.runInteraction( - "test_list_missing_replica_identities", - _list_tables_with_missing_replica_identities_txn, - ) - ) - self.assertEqual( - len(missing), - 0, - f"The following tables in the {pool.name()!r} database are missing REPLICA IDENTITIES: {missing!r}.", - )