diff --git a/CHANGES.md b/CHANGES.md index be5c18c84ba..31d9914b39f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,50 @@ +# Synapse 1.120.0rc1 (2024-11-20) + +This release enables the enforcement of authenticated media by default, with exemptions for media that is already present in the +homeserver's media store. + +Most homeservers operating in the public federation will not be impacted by this change, given that +the large homeserver `matrix.org` enabled this in September 2024 and therefore most clients and servers +will already have updated as a result. + +Some server administrators may still wish to disable this enforcement for the time being, in the interest of compatibility with older clients +and older federated homeservers. +See the [upgrade notes](https://element-hq.github.io/synapse/v1.120/upgrade.html#authenticated-media-is-now-enforced-by-default) for more information. + +### Features + +- Enforce authenticated media by default. Administrators can revert this by configuring `enable_authenticated_media` to `false`. In a future release of Synapse, this option will be removed and become always-on. ([\#17889](https://github.com/element-hq/synapse/issues/17889)) +- Add a one-off task to delete old One-Time Keys, to guard against us having old OTKs in the database that the client has long forgotten about. ([\#17934](https://github.com/element-hq/synapse/issues/17934)) + +### Improved Documentation + +- Clarify the semantics of the `enable_authenticated_media` configuration option. ([\#17913](https://github.com/element-hq/synapse/issues/17913)) +- Add documentation about backing up Synapse. ([\#17931](https://github.com/element-hq/synapse/issues/17931)) + +### Deprecations and Removals + +- Remove support for [MSC3886: Simple client rendezvous capability](https://github.com/matrix-org/matrix-spec-proposals/pull/3886), which has been superseded by [MSC4108](https://github.com/matrix-org/matrix-spec-proposals/pull/4108) and therefore closed. ([\#17638](https://github.com/element-hq/synapse/issues/17638)) + +### Internal Changes + +- Addressed some typos in docs and returned error message for unknown MXC ID. ([\#17865](https://github.com/element-hq/synapse/issues/17865)) +- Unpin the upload release GHA action. ([\#17923](https://github.com/element-hq/synapse/issues/17923)) +- Bump macOS version used to build wheels during release, as current version used is end-of-life. ([\#17924](https://github.com/element-hq/synapse/issues/17924)) +- Move server event filtering logic to Rust. ([\#17928](https://github.com/element-hq/synapse/issues/17928)) +- Support new package name of PyPI package `python-multipart` 0.0.13 so that distro packagers do not need to work around name conflict with PyPI package `multipart`. ([\#17932](https://github.com/element-hq/synapse/issues/17932)) +- Speed up slow initial sliding syncs on large servers. ([\#17946](https://github.com/element-hq/synapse/issues/17946)) + +### Updates to locked dependencies + +* Bump anyhow from 1.0.92 to 1.0.93. ([\#17920](https://github.com/element-hq/synapse/issues/17920)) +* Bump bleach from 6.1.0 to 6.2.0. ([\#17918](https://github.com/element-hq/synapse/issues/17918)) +* Bump immutabledict from 4.2.0 to 4.2.1. ([\#17941](https://github.com/element-hq/synapse/issues/17941)) +* Bump packaging from 24.1 to 24.2. ([\#17940](https://github.com/element-hq/synapse/issues/17940)) +* Bump phonenumbers from 8.13.49 to 8.13.50. ([\#17942](https://github.com/element-hq/synapse/issues/17942)) +* Bump pygithub from 2.4.0 to 2.5.0. ([\#17917](https://github.com/element-hq/synapse/issues/17917)) +* Bump ruff from 0.7.2 to 0.7.3. ([\#17919](https://github.com/element-hq/synapse/issues/17919)) +* Bump serde from 1.0.214 to 1.0.215. ([\#17938](https://github.com/element-hq/synapse/issues/17938)) + # Synapse 1.119.0 (2024-11-13) No significant changes since 1.119.0rc2. diff --git a/Cargo.lock b/Cargo.lock index 46c930ebd75..5c8f627fd7c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -505,9 +505,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.132" +version = "1.0.133" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d726bfaff4b320266d395898905d0eba0345aae23b54aee3a737e260fd46db03" +checksum = "c7fceb2473b9166b2294ef05efcb65a3db80803f0b03ef86a5fc88a2b85ee377" dependencies = [ "itoa", "memchr", diff --git a/changelog.d/17638.removal b/changelog.d/17638.removal deleted file mode 100644 index 1bb09e976ec..00000000000 --- a/changelog.d/17638.removal +++ /dev/null @@ -1 +0,0 @@ -Remove support for closed [MSC3886](https://github.com/matrix-org/matrix-spec-proposals/pull/3886). \ No newline at end of file diff --git a/changelog.d/17865.misc b/changelog.d/17865.misc deleted file mode 100644 index 2303a7e1b74..00000000000 --- a/changelog.d/17865.misc +++ /dev/null @@ -1 +0,0 @@ -Addressed some typos in docs and returned error message for unknown MXC ID. diff --git a/changelog.d/17872.doc b/changelog.d/17872.doc new file mode 100644 index 00000000000..7f8b2d34953 --- /dev/null +++ b/changelog.d/17872.doc @@ -0,0 +1 @@ +Add OIDC example configuration for Forgejo (fork of Gitea). diff --git a/changelog.d/17913.doc b/changelog.d/17913.doc deleted file mode 100644 index 39f59795625..00000000000 --- a/changelog.d/17913.doc +++ /dev/null @@ -1 +0,0 @@ -Clarify the semantics of the `enable_authenticated_media` configuration option. diff --git a/changelog.d/17923.misc b/changelog.d/17923.misc deleted file mode 100644 index 4d74e7e1849..00000000000 --- a/changelog.d/17923.misc +++ /dev/null @@ -1 +0,0 @@ -Unpin the upload release GHA action. diff --git a/changelog.d/17924.misc b/changelog.d/17924.misc deleted file mode 100644 index c7cc502360e..00000000000 --- a/changelog.d/17924.misc +++ /dev/null @@ -1 +0,0 @@ -Bump macos version used to build wheels during release, as current version used is end-of-life. diff --git a/changelog.d/17928.misc b/changelog.d/17928.misc deleted file mode 100644 index b5aef4457a4..00000000000 --- a/changelog.d/17928.misc +++ /dev/null @@ -1 +0,0 @@ -Move server event filtering logic to rust. diff --git a/changelog.d/17931.doc b/changelog.d/17931.doc deleted file mode 100644 index 9207cb0a1c5..00000000000 --- a/changelog.d/17931.doc +++ /dev/null @@ -1 +0,0 @@ -Add documentation about backing up Synapse. diff --git a/changelog.d/17934.feature b/changelog.d/17934.feature deleted file mode 100644 index f0e138a30ff..00000000000 --- a/changelog.d/17934.feature +++ /dev/null @@ -1 +0,0 @@ -Add a one-off task to delete old one-time-keys, to guard against us having old OTKs in the database that the client has long forgotten about. diff --git a/changelog.d/17936.misc b/changelog.d/17936.misc new file mode 100644 index 00000000000..91d976fbd9c --- /dev/null +++ b/changelog.d/17936.misc @@ -0,0 +1 @@ +Fix incorrect comment in new schema delta. diff --git a/changelog.d/17944.misc b/changelog.d/17944.misc new file mode 100644 index 00000000000..a8a645103f4 --- /dev/null +++ b/changelog.d/17944.misc @@ -0,0 +1 @@ +Raise setuptools_rust version cap to 1.10.2. \ No newline at end of file diff --git a/changelog.d/17945.misc b/changelog.d/17945.misc new file mode 100644 index 00000000000..eeebb921699 --- /dev/null +++ b/changelog.d/17945.misc @@ -0,0 +1 @@ +Enable encrypted appservice related experimental features in the complement docker image. diff --git a/changelog.d/17952.misc b/changelog.d/17952.misc new file mode 100644 index 00000000000..84fc8bfc290 --- /dev/null +++ b/changelog.d/17952.misc @@ -0,0 +1 @@ +Return whether the user is suspended when querying the user account in the Admin API. \ No newline at end of file diff --git a/changelog.d/17953.doc b/changelog.d/17953.doc new file mode 100644 index 00000000000..10f5a27ba9f --- /dev/null +++ b/changelog.d/17953.doc @@ -0,0 +1 @@ +Link to element-docker-demo from contrib/docker*. diff --git a/contrib/docker/README.md b/contrib/docker/README.md index 89c1518bd0b..fdfa96795a1 100644 --- a/contrib/docker/README.md +++ b/contrib/docker/README.md @@ -30,3 +30,6 @@ docker-compose up -d ### More information For more information on required environment variables and mounts, see the main docker documentation at [/docker/README.md](../../docker/README.md) + +**For a more comprehensive Docker Compose example showcasing a full Matrix 2.0 stack, please see +https://github.com/element-hq/element-docker-demo** \ No newline at end of file diff --git a/contrib/docker_compose_workers/README.md b/contrib/docker_compose_workers/README.md index 81518f6ba12..16c8c26795d 100644 --- a/contrib/docker_compose_workers/README.md +++ b/contrib/docker_compose_workers/README.md @@ -8,6 +8,9 @@ All examples and snippets assume that your Synapse service is called `synapse` i An example Docker Compose file can be found [here](docker-compose.yaml). +**For a more comprehensive Docker Compose example, showcasing a full Matrix 2.0 stack (originally based on this +docker-compose.yaml), please see https://github.com/element-hq/element-docker-demo** + ## Worker Service Examples in Docker Compose In order to start the Synapse container as a worker, you must specify an `entrypoint` that loads both the `homeserver.yaml` and the configuration for the worker (`synapse-generic-worker-1.yaml` in the example below). You must also include the worker type in the environment variable `SYNAPSE_WORKER` or alternatively pass `-m synapse.app.generic_worker` as part of the `entrypoint` after `"/start.py", "run"`). diff --git a/debian/changelog b/debian/changelog index bacd453cb4f..d7cec3fa8a8 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.120.0~rc1) stable; urgency=medium + + * New Synapse release 1.120.0rc1. + + -- Synapse Packaging team Wed, 20 Nov 2024 15:02:21 +0000 + matrix-synapse-py3 (1.119.0) stable; urgency=medium * New Synapse release 1.119.0. diff --git a/docker/complement/conf/workers-shared-extra.yaml.j2 b/docker/complement/conf/workers-shared-extra.yaml.j2 index b9334cc53bb..9a74c617bc4 100644 --- a/docker/complement/conf/workers-shared-extra.yaml.j2 +++ b/docker/complement/conf/workers-shared-extra.yaml.j2 @@ -104,6 +104,16 @@ experimental_features: msc3967_enabled: true # Expose a room summary for public rooms msc3266_enabled: true + # Send to-device messages to application services + msc2409_to_device_messages_enabled: true + # Allow application services to masquerade devices + msc3202_device_masquerading: true + # Sending device list changes, one-time key counts and fallback key usage to application services + msc3202_transaction_extensions: true + # Proxy OTK claim requests to exclusive ASes + msc3983_appservice_otk_claims: true + # Proxy key queries to exclusive ASes + msc3984_appservice_key_query: true server_notices: system_mxid_localpart: _server diff --git a/docs/admin_api/user_admin_api.md b/docs/admin_api/user_admin_api.md index 96a2994b7b4..a6e2e0a1537 100644 --- a/docs/admin_api/user_admin_api.md +++ b/docs/admin_api/user_admin_api.md @@ -55,7 +55,8 @@ It returns a JSON body like the following: } ], "user_type": null, - "locked": false + "locked": false, + "suspended": false } ``` diff --git a/docs/openid.md b/docs/openid.md index 7a10b1615b8..5a3d7e9fba6 100644 --- a/docs/openid.md +++ b/docs/openid.md @@ -336,6 +336,36 @@ but it has a `response_types_supported` which excludes "code" (which we rely on, is even mentioned in their [documentation](https://developers.facebook.com/docs/facebook-login/manually-build-a-login-flow#login)), so we have to disable discovery and configure the URIs manually. +### Forgejo + +Forgejo is a fork of Gitea that can act as an OAuth2 provider. + +The implementation of OAuth2 is improved compared to Gitea, as it provides a correctly defined `subject_claim` and `scopes`. + +Synapse config: + +```yaml +oidc_providers: + - idp_id: forgejo + idp_name: Forgejo + discover: false + issuer: "https://your-forgejo.com/" + client_id: "your-client-id" # TO BE FILLED + client_secret: "your-client-secret" # TO BE FILLED + client_auth_method: client_secret_post + scopes: ["openid", "profile", "email", "groups"] + authorization_endpoint: "https://your-forgejo.com/login/oauth/authorize" + token_endpoint: "https://your-forgejo.com/login/oauth/access_token" + userinfo_endpoint: "https://your-forgejo.com/api/v1/user" + user_mapping_provider: + config: + subject_claim: "sub" + picture_claim: "picture" + localpart_template: "{{ user.preferred_username }}" + display_name_template: "{{ user.name }}" + email_template: "{{ user.email }}" +``` + ### GitHub [GitHub][github-idp] is a bit special as it is not an OpenID Connect compliant provider, but diff --git a/docs/upgrade.md b/docs/upgrade.md index 9f12d7c34f3..45e63b0c5de 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -128,6 +128,29 @@ removing the experimental support for it in this release. The `experimental_features.msc3886_endpoint` configuration option has been removed. +## Authenticated media is now enforced by default + +The [`enable_authenticated_media`] configuration option now defaults to true. + +This means that clients and remote (federated) homeservers now need to use +the authenticated media endpoints in order to download media from your +homeserver. + +As an exception, existing media that was stored on the server prior to +this option changing to `true` will still be accessible over the +unauthenticated endpoints. + +The matrix.org homeserver has already been running with this option enabled +since September 2024, so most common clients and homeservers should already +be compatible. + +With that said, administrators who wish to disable this feature for broader +compatibility can still do so by manually configuring +`enable_authenticated_media: False`. + +[`enable_authenticated_media`]: usage/configuration/config_documentation.md#enable_authenticated_media + + # Upgrading to v1.119.0 ## Minimum supported Python version diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index 005633e46bb..7a48d76bbb1 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -1887,8 +1887,7 @@ Config options related to Synapse's media store. When set to true, all subsequent media uploads will be marked as authenticated, and will not be available over legacy unauthenticated media endpoints (`/_matrix/media/(r0|v3|v1)/download` and `/_matrix/media/(r0|v3|v1)/thumbnail`) - requests for authenticated media over these endpoints will result in a 404. All media, including authenticated media, will be available over the authenticated media endpoints `_matrix/client/v1/media/download` and `_matrix/client/v1/media/thumbnail`. Media uploaded prior to setting this option to true will still be available over the legacy endpoints. Note if the setting is switched to false -after enabling, media marked as authenticated will be available over legacy endpoints. Defaults to false, but -this will change to true in a future Synapse release. +after enabling, media marked as authenticated will be available over legacy endpoints. Defaults to true (previously false). In a future release of Synapse, this option will be removed and become always-on. In all cases, authenticated requests to download media will succeed, but for unauthenticated requests, this case-by-case breakdown describes whether media downloads are permitted: @@ -1910,9 +1909,11 @@ will perpetually be available over the legacy, unauthenticated endpoint, even af This is for backwards compatibility with older clients and homeservers that do not yet support requesting authenticated media; those older clients or homeservers will not be cut off from media they can already see. +_Changed in Synapse 1.120:_ This option now defaults to `True` when not set, whereas before this version it defaulted to `False`. + Example configuration: ```yaml -enable_authenticated_media: true +enable_authenticated_media: false ``` --- ### `enable_media_repo` diff --git a/flake.lock b/flake.lock index 6b25cef3fc9..a6a2aea3285 100644 --- a/flake.lock +++ b/flake.lock @@ -56,24 +56,6 @@ "type": "github" } }, - "flake-utils_2": { - "inputs": { - "systems": "systems_2" - }, - "locked": { - "lastModified": 1681202837, - "narHash": "sha256-H+Rh19JDwRtpVPAWp64F+rlEtxUWBAQW28eAi3SRSzg=", - "owner": "numtide", - "repo": "flake-utils", - "rev": "cfacdce06f30d2b68473a46042957675eebb3401", - "type": "github" - }, - "original": { - "owner": "numtide", - "repo": "flake-utils", - "type": "github" - } - }, "gitignore": { "inputs": { "nixpkgs": [ @@ -202,11 +184,11 @@ }, "nixpkgs_3": { "locked": { - "lastModified": 1681358109, - "narHash": "sha256-eKyxW4OohHQx9Urxi7TQlFBTDWII+F+x2hklDOQPB50=", + "lastModified": 1728538411, + "narHash": "sha256-f0SBJz1eZ2yOuKUr5CA9BHULGXVSn6miBuUWdTyhUhU=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "96ba1c52e54e74c3197f4d43026b3f3d92e83ff9", + "rev": "b69de56fac8c2b6f8fd27f2eca01dcda8e0a4221", "type": "github" }, "original": { @@ -249,20 +231,19 @@ "devenv": "devenv", "nixpkgs": "nixpkgs_2", "rust-overlay": "rust-overlay", - "systems": "systems_3" + "systems": "systems_2" } }, "rust-overlay": { "inputs": { - "flake-utils": "flake-utils_2", "nixpkgs": "nixpkgs_3" }, "locked": { - "lastModified": 1693966243, - "narHash": "sha256-a2CA1aMIPE67JWSVIGoGtD3EGlFdK9+OlJQs0FOWCKY=", + "lastModified": 1731897198, + "narHash": "sha256-Ou7vLETSKwmE/HRQz4cImXXJBr/k9gp4J4z/PF8LzTE=", "owner": "oxalica", "repo": "rust-overlay", - "rev": "a8b4bb4cbb744baaabc3e69099f352f99164e2c1", + "rev": "0be641045af6d8666c11c2c40e45ffc9667839b5", "type": "github" }, "original": { @@ -300,21 +281,6 @@ "repo": "default", "type": "github" } - }, - "systems_3": { - "locked": { - "lastModified": 1681028828, - "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", - "owner": "nix-systems", - "repo": "default", - "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", - "type": "github" - }, - "original": { - "owner": "nix-systems", - "repo": "default", - "type": "github" - } } }, "root": "root", diff --git a/flake.nix b/flake.nix index bc360ae44a2..749c10da1d0 100644 --- a/flake.nix +++ b/flake.nix @@ -82,7 +82,7 @@ # # NOTE: We currently need to set the Rust version unnecessarily high # in order to work around https://github.com/matrix-org/synapse/issues/15939 - (rust-bin.stable."1.71.1".default.override { + (rust-bin.stable."1.82.0".default.override { # Additionally install the "rust-src" extension to allow diving into the # Rust source code in an IDE (rust-analyzer will also make use of it). extensions = [ "rust-src" ]; @@ -205,7 +205,7 @@ # corresponding Nix packages on https://search.nixos.org/packages. # # This was done until `./install-deps.pl --dryrun` produced no output. - env.PERL5LIB = "${with pkgs.perl536Packages; makePerlPath [ + env.PERL5LIB = "${with pkgs.perl538Packages; makePerlPath [ DBI ClassMethodModifiers CryptEd25519 diff --git a/pyproject.toml b/pyproject.toml index bb2422da6fc..2e52988c037 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -97,7 +97,7 @@ module-name = "synapse.synapse_rust" [tool.poetry] name = "matrix-synapse" -version = "1.119.0" +version = "1.120.0rc1" description = "Homeserver for the Matrix decentralised comms protocol" authors = ["Matrix.org Team and Contributors "] license = "AGPL-3.0-or-later" @@ -371,7 +371,7 @@ tomli = ">=1.2.3" # runtime errors caused by build system changes. # We are happy to raise these upper bounds upon request, # provided we check that it's safe to do so (i.e. that CI passes). -requires = ["poetry-core>=1.1.0,<=1.9.1", "setuptools_rust>=1.3,<=1.8.1"] +requires = ["poetry-core>=1.1.0,<=1.9.1", "setuptools_rust>=1.3,<=1.10.2"] build-backend = "poetry.core.masonry.api" diff --git a/synapse/_scripts/synapse_port_db.py b/synapse/_scripts/synapse_port_db.py index 31639d366ec..d8f6f8ebdc3 100755 --- a/synapse/_scripts/synapse_port_db.py +++ b/synapse/_scripts/synapse_port_db.py @@ -88,6 +88,7 @@ from synapse.storage.databases.main.room import RoomBackgroundUpdateStore from synapse.storage.databases.main.roommember import RoomMemberBackgroundUpdateStore from synapse.storage.databases.main.search import SearchBackgroundUpdateStore +from synapse.storage.databases.main.sliding_sync import SlidingSyncStore from synapse.storage.databases.main.state import MainStateBackgroundUpdateStore from synapse.storage.databases.main.stats import StatsStore from synapse.storage.databases.main.user_directory import ( @@ -255,6 +256,7 @@ class Store( ReceiptsBackgroundUpdateStore, RelationsWorkerStore, EventFederationWorkerStore, + SlidingSyncStore, ): def execute(self, f: Callable[..., R], *args: Any, **kwargs: Any) -> Awaitable[R]: return self.db_pool.runInteraction(f.__name__, f, *args, **kwargs) diff --git a/synapse/config/repository.py b/synapse/config/repository.py index 97ce6de5281..27860154e1c 100644 --- a/synapse/config/repository.py +++ b/synapse/config/repository.py @@ -272,9 +272,7 @@ def read_config(self, config: JsonDict, **kwargs: Any) -> None: remote_media_lifetime ) - self.enable_authenticated_media = config.get( - "enable_authenticated_media", False - ) + self.enable_authenticated_media = config.get("enable_authenticated_media", True) def generate_config_section(self, data_dir_path: str, **kwargs: Any) -> str: assert data_dir_path is not None diff --git a/synapse/handlers/admin.py b/synapse/handlers/admin.py index d1989e9d2c6..d1194545aeb 100644 --- a/synapse/handlers/admin.py +++ b/synapse/handlers/admin.py @@ -124,6 +124,7 @@ async def get_user(self, user: UserID) -> Optional[JsonMapping]: "consent_ts": user_info.consent_ts, "user_type": user_info.user_type, "is_guest": user_info.is_guest, + "suspended": user_info.suspended, } if self._msc3866_enabled: diff --git a/synapse/http/client.py b/synapse/http/client.py index c3b2299c954..85923d956bc 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -36,7 +36,6 @@ ) import attr -import multipart import treq from canonicaljson import encode_canonical_json from netaddr import AddrFormatError, IPAddress, IPSet @@ -93,6 +92,20 @@ if TYPE_CHECKING: from synapse.server import HomeServer +# Support both import names for the `python-multipart` (PyPI) library, +# which renamed its package name from `multipart` to `python_multipart` +# in 0.0.13 (though supports the old import name for compatibility). +# Note that the `multipart` package name conflicts with `multipart` (PyPI) +# so we should prefer importing from `python_multipart` when possible. +try: + from python_multipart import MultipartParser + + if TYPE_CHECKING: + from python_multipart import multipart +except ImportError: + from multipart import MultipartParser # type: ignore[no-redef] + + logger = logging.getLogger(__name__) outgoing_requests_counter = Counter("synapse_http_client_requests", "", ["method"]) @@ -1039,7 +1052,7 @@ def __init__( self.deferred = deferred self.boundary = boundary self.max_length = max_length - self.parser: Optional[multipart.MultipartParser] = None + self.parser: Optional[MultipartParser] = None self.multipart_response = MultipartResponse() self.has_redirect = False self.in_json = False @@ -1097,12 +1110,12 @@ def on_part_data(data: bytes, start: int, end: int) -> None: self.deferred.errback() self.file_length += end - start - callbacks: "multipart.multipart.MultipartCallbacks" = { + callbacks: "multipart.MultipartCallbacks" = { "on_header_field": on_header_field, "on_header_value": on_header_value, "on_part_data": on_part_data, } - self.parser = multipart.MultipartParser(self.boundary, callbacks) + self.parser = MultipartParser(self.boundary, callbacks) self.total_length += len(incoming_data) if self.max_length is not None and self.total_length >= self.max_length: diff --git a/synapse/storage/databases/main/sliding_sync.py b/synapse/storage/databases/main/sliding_sync.py index 7b357c1ffea..874dfdcb77a 100644 --- a/synapse/storage/databases/main/sliding_sync.py +++ b/synapse/storage/databases/main/sliding_sync.py @@ -21,7 +21,11 @@ from synapse.api.errors import SlidingSyncUnknownPosition from synapse.logging.opentracing import log_kv from synapse.storage._base import SQLBaseStore, db_to_json -from synapse.storage.database import LoggingTransaction +from synapse.storage.database import ( + DatabasePool, + LoggingDatabaseConnection, + LoggingTransaction, +) from synapse.types import MultiWriterStreamToken, RoomStreamToken from synapse.types.handlers.sliding_sync import ( HaveSentRoom, @@ -35,12 +39,28 @@ from synapse.util.caches.descriptors import cached if TYPE_CHECKING: + from synapse.server import HomeServer from synapse.storage.databases.main import DataStore logger = logging.getLogger(__name__) class SlidingSyncStore(SQLBaseStore): + def __init__( + self, + database: DatabasePool, + db_conn: LoggingDatabaseConnection, + hs: "HomeServer", + ): + super().__init__(database, db_conn, hs) + + self.db_pool.updates.register_background_index_update( + update_name="sliding_sync_connection_room_configs_required_state_id_idx", + index_name="sliding_sync_connection_room_configs_required_state_id_idx", + table="sliding_sync_connection_room_configs", + columns=("required_state_id",), + ) + async def get_latest_bump_stamp_for_room( self, room_id: str, diff --git a/synapse/storage/schema/main/delta/88/04_current_state_delta_index.sql b/synapse/storage/schema/main/delta/88/04_current_state_delta_index.sql index ad54302a8f9..0ee78df1a0a 100644 --- a/synapse/storage/schema/main/delta/88/04_current_state_delta_index.sql +++ b/synapse/storage/schema/main/delta/88/04_current_state_delta_index.sql @@ -12,7 +12,7 @@ -- . --- Add an index on (user_id, device_id, algorithm, ts_added_ms) on e2e_one_time_keys_json, so that OTKs can --- efficiently be issued in the same order they were uploaded. +-- Add an index on `current_state_delta_stream(room_id, stream_id)` to allow +-- efficient per-room lookups. INSERT INTO background_updates (ordering, update_name, progress_json) VALUES (8804, 'current_state_delta_stream_room_index', '{}'); diff --git a/synapse/storage/schema/main/delta/88/05_sliding_sync_room_config_index.sql b/synapse/storage/schema/main/delta/88/05_sliding_sync_room_config_index.sql new file mode 100644 index 00000000000..7b2e18a84bc --- /dev/null +++ b/synapse/storage/schema/main/delta/88/05_sliding_sync_room_config_index.sql @@ -0,0 +1,20 @@ +-- +-- This file is licensed under the Affero General Public License (AGPL) version 3. +-- +-- Copyright (C) 2024 New Vector, Ltd +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- See the GNU Affero General Public License for more details: +-- . + + +-- Add an index on sliding_sync_connection_room_configs(required_state_id), so +-- that when we delete entries in `sliding_sync_connection_required_state` it's +-- efficient for Postgres to check they've been deleted from +-- `sliding_sync_connection_room_configs` too +INSERT INTO background_updates (ordering, update_name, progress_json) VALUES + (8805, 'sliding_sync_connection_room_configs_required_state_id_idx', '{}'); diff --git a/tests/media/test_media_storage.py b/tests/media/test_media_storage.py index 034d9ece0b5..f4fbc0544a2 100644 --- a/tests/media/test_media_storage.py +++ b/tests/media/test_media_storage.py @@ -419,6 +419,11 @@ def _req( return channel + @unittest.override_config( + { + "enable_authenticated_media": False, + } + ) def test_handle_missing_content_type(self) -> None: channel = self._req( b"attachment; filename=out" + self.test_image.extension, @@ -430,6 +435,11 @@ def test_handle_missing_content_type(self) -> None: headers.getRawHeaders(b"Content-Type"), [b"application/octet-stream"] ) + @unittest.override_config( + { + "enable_authenticated_media": False, + } + ) def test_disposition_filename_ascii(self) -> None: """ If the filename is filename= then Synapse will decode it as an @@ -450,6 +460,11 @@ def test_disposition_filename_ascii(self) -> None: ], ) + @unittest.override_config( + { + "enable_authenticated_media": False, + } + ) def test_disposition_filenamestar_utf8escaped(self) -> None: """ If the filename is filename=*utf8'' then Synapse will @@ -475,6 +490,11 @@ def test_disposition_filenamestar_utf8escaped(self) -> None: ], ) + @unittest.override_config( + { + "enable_authenticated_media": False, + } + ) def test_disposition_none(self) -> None: """ If there is no filename, Content-Disposition should only @@ -491,6 +511,11 @@ def test_disposition_none(self) -> None: [b"inline" if self.test_image.is_inline else b"attachment"], ) + @unittest.override_config( + { + "enable_authenticated_media": False, + } + ) def test_thumbnail_crop(self) -> None: """Test that a cropped remote thumbnail is available.""" self._test_thumbnail( @@ -500,6 +525,11 @@ def test_thumbnail_crop(self) -> None: unable_to_thumbnail=self.test_image.unable_to_thumbnail, ) + @unittest.override_config( + { + "enable_authenticated_media": False, + } + ) def test_thumbnail_scale(self) -> None: """Test that a scaled remote thumbnail is available.""" self._test_thumbnail( @@ -509,6 +539,11 @@ def test_thumbnail_scale(self) -> None: unable_to_thumbnail=self.test_image.unable_to_thumbnail, ) + @unittest.override_config( + { + "enable_authenticated_media": False, + } + ) def test_invalid_type(self) -> None: """An invalid thumbnail type is never available.""" self._test_thumbnail( @@ -519,7 +554,10 @@ def test_invalid_type(self) -> None: ) @unittest.override_config( - {"thumbnail_sizes": [{"width": 32, "height": 32, "method": "scale"}]} + { + "thumbnail_sizes": [{"width": 32, "height": 32, "method": "scale"}], + "enable_authenticated_media": False, + }, ) def test_no_thumbnail_crop(self) -> None: """ @@ -533,7 +571,10 @@ def test_no_thumbnail_crop(self) -> None: ) @unittest.override_config( - {"thumbnail_sizes": [{"width": 32, "height": 32, "method": "crop"}]} + { + "thumbnail_sizes": [{"width": 32, "height": 32, "method": "crop"}], + "enable_authenticated_media": False, + } ) def test_no_thumbnail_scale(self) -> None: """ @@ -546,6 +587,11 @@ def test_no_thumbnail_scale(self) -> None: unable_to_thumbnail=self.test_image.unable_to_thumbnail, ) + @unittest.override_config( + { + "enable_authenticated_media": False, + } + ) def test_thumbnail_repeated_thumbnail(self) -> None: """Test that fetching the same thumbnail works, and deleting the on disk thumbnail regenerates it. @@ -720,6 +766,11 @@ def test_same_quality(self, method: str, desired_size: int) -> None: ) ) + @unittest.override_config( + { + "enable_authenticated_media": False, + } + ) def test_x_robots_tag_header(self) -> None: """ Tests that the `X-Robots-Tag` header is present, which informs web crawlers @@ -733,6 +784,11 @@ def test_x_robots_tag_header(self) -> None: [b"noindex, nofollow, noarchive, noimageindex"], ) + @unittest.override_config( + { + "enable_authenticated_media": False, + } + ) def test_cross_origin_resource_policy_header(self) -> None: """ Test that the Cross-Origin-Resource-Policy header is set to "cross-origin" @@ -747,6 +803,11 @@ def test_cross_origin_resource_policy_header(self) -> None: [b"cross-origin"], ) + @unittest.override_config( + { + "enable_authenticated_media": False, + } + ) def test_unknown_v3_endpoint(self) -> None: """ If the v3 endpoint fails, try the r0 one. @@ -985,6 +1046,11 @@ def read_body_with_max_size_50MiB(*args: Any, **kwargs: Any) -> Deferred: d.callback(52428800) return d + @override_config( + { + "enable_authenticated_media": False, + } + ) @patch( "synapse.http.matrixfederationclient.read_body_with_max_size", read_body_with_max_size_30MiB, @@ -1060,6 +1126,7 @@ async def _send_request(*args: Any, **kwargs: Any) -> IResponse: { "remote_media_download_per_second": "50M", "remote_media_download_burst_count": "50M", + "enable_authenticated_media": False, } ) @patch( @@ -1119,7 +1186,12 @@ async def _send_request(*args: Any, **kwargs: Any) -> IResponse: ) assert channel.code == 200 - @override_config({"remote_media_download_burst_count": "87M"}) + @override_config( + { + "remote_media_download_burst_count": "87M", + "enable_authenticated_media": False, + } + ) @patch( "synapse.http.matrixfederationclient.read_body_with_max_size", read_body_with_max_size_30MiB, @@ -1159,7 +1231,7 @@ async def _send_request(*args: Any, **kwargs: Any) -> IResponse: ) assert channel2.code == 429 - @override_config({"max_upload_size": "29M"}) + @override_config({"max_upload_size": "29M", "enable_authenticated_media": False}) @patch( "synapse.http.matrixfederationclient.read_body_with_max_size", read_body_with_max_size_30MiB, diff --git a/tests/replication/test_multi_media_repo.py b/tests/replication/test_multi_media_repo.py index 6fc4600c41b..f36af877c4e 100644 --- a/tests/replication/test_multi_media_repo.py +++ b/tests/replication/test_multi_media_repo.py @@ -40,6 +40,7 @@ from tests.replication._base import BaseMultiWorkerStreamTestCase from tests.server import FakeChannel, FakeTransport, make_request from tests.test_utils import SMALL_PNG +from tests.unittest import override_config logger = logging.getLogger(__name__) @@ -148,6 +149,7 @@ def _get_media_req( return channel, request + @override_config({"enable_authenticated_media": False}) def test_basic(self) -> None: """Test basic fetching of remote media from a single worker.""" hs1 = self.make_worker_hs("synapse.app.generic_worker") @@ -164,6 +166,7 @@ def test_basic(self) -> None: self.assertEqual(channel.code, 200) self.assertEqual(channel.result["body"], b"Hello!") + @override_config({"enable_authenticated_media": False}) def test_download_simple_file_race(self) -> None: """Test that fetching remote media from two different processes at the same time works. @@ -203,6 +206,7 @@ def test_download_simple_file_race(self) -> None: # We expect only one new file to have been persisted. self.assertEqual(start_count + 1, self._count_remote_media()) + @override_config({"enable_authenticated_media": False}) def test_download_image_race(self) -> None: """Test that fetching remote *images* from two different processes at the same time works. diff --git a/tests/rest/admin/test_admin.py b/tests/rest/admin/test_admin.py index 6351326fff8..5483f8f37f6 100644 --- a/tests/rest/admin/test_admin.py +++ b/tests/rest/admin/test_admin.py @@ -30,7 +30,7 @@ import synapse.rest.admin from synapse.http.server import JsonResource from synapse.rest.admin import VersionServlet -from synapse.rest.client import login, room +from synapse.rest.client import login, media, room from synapse.server import HomeServer from synapse.util import Clock @@ -60,6 +60,7 @@ class QuarantineMediaTestCase(unittest.HomeserverTestCase): synapse.rest.admin.register_servlets, synapse.rest.admin.register_servlets_for_media_repo, login.register_servlets, + media.register_servlets, room.register_servlets, ] @@ -74,7 +75,7 @@ def _ensure_quarantined( """Ensure a piece of media is quarantined when trying to access it.""" channel = self.make_request( "GET", - f"/_matrix/media/v3/download/{server_and_media_id}", + f"/_matrix/client/v1/media/download/{server_and_media_id}", shorthand=False, access_token=admin_user_tok, ) @@ -131,7 +132,7 @@ def test_quarantine_media_by_id(self) -> None: # Attempt to access the media channel = self.make_request( "GET", - f"/_matrix/media/v3/download/{server_name_and_media_id}", + f"/_matrix/client/v1/media/download/{server_name_and_media_id}", shorthand=False, access_token=non_admin_user_tok, ) @@ -295,7 +296,7 @@ def test_cannot_quarantine_safe_media(self) -> None: # Attempt to access each piece of media channel = self.make_request( "GET", - f"/_matrix/media/v3/download/{server_and_media_id_2}", + f"/_matrix/client/v1/media/download/{server_and_media_id_2}", shorthand=False, access_token=non_admin_user_tok, ) diff --git a/tests/rest/admin/test_media.py b/tests/rest/admin/test_media.py index f3781655132..19c244cfcfc 100644 --- a/tests/rest/admin/test_media.py +++ b/tests/rest/admin/test_media.py @@ -36,6 +36,7 @@ from tests import unittest from tests.test_utils import SMALL_PNG +from tests.unittest import override_config VALID_TIMESTAMP = 1609459200000 # 2021-01-01 in milliseconds INVALID_TIMESTAMP_IN_S = 1893456000 # 2030-01-01 in seconds @@ -126,6 +127,7 @@ def test_media_is_not_local(self) -> None: self.assertEqual(400, channel.code, msg=channel.json_body) self.assertEqual("Can only delete local media", channel.json_body["error"]) + @override_config({"enable_authenticated_media": False}) def test_delete_media(self) -> None: """ Tests that delete a media is successfully @@ -371,6 +373,7 @@ def test_delete_media_never_accessed(self, use_legacy_url: bool) -> None: self._access_media(server_and_media_id, False) + @override_config({"enable_authenticated_media": False}) def test_keep_media_by_date(self) -> None: """ Tests that media is not deleted if it is newer than `before_ts` @@ -408,6 +411,7 @@ def test_keep_media_by_date(self) -> None: self._access_media(server_and_media_id, False) + @override_config({"enable_authenticated_media": False}) def test_keep_media_by_size(self) -> None: """ Tests that media is not deleted if its size is smaller than or equal @@ -443,6 +447,7 @@ def test_keep_media_by_size(self) -> None: self._access_media(server_and_media_id, False) + @override_config({"enable_authenticated_media": False}) def test_keep_media_by_user_avatar(self) -> None: """ Tests that we do not delete media if is used as a user avatar @@ -487,6 +492,7 @@ def test_keep_media_by_user_avatar(self) -> None: self._access_media(server_and_media_id, False) + @override_config({"enable_authenticated_media": False}) def test_keep_media_by_room_avatar(self) -> None: """ Tests that we do not delete media if it is used as a room avatar diff --git a/tests/rest/admin/test_user.py b/tests/rest/admin/test_user.py index 668ccb89ff1..fdb8fafa0e8 100644 --- a/tests/rest/admin/test_user.py +++ b/tests/rest/admin/test_user.py @@ -45,6 +45,7 @@ devices, login, logout, + media, profile, register, room, @@ -3221,6 +3222,7 @@ def _check_fields(self, content: JsonDict) -> None: self.assertIn("consent_ts", content) self.assertIn("external_ids", content) self.assertIn("last_seen_ts", content) + self.assertIn("suspended", content) # This key was removed intentionally. Ensure it is not accidentally re-included. self.assertNotIn("password_hash", content) @@ -3517,6 +3519,7 @@ class UserMediaRestTestCase(unittest.HomeserverTestCase): servlets = [ synapse.rest.admin.register_servlets, login.register_servlets, + media.register_servlets, ] def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: @@ -4023,7 +4026,7 @@ def _create_media_and_access( # Try to access a media and to create `last_access_ts` channel = self.make_request( "GET", - f"/_matrix/media/v3/download/{server_and_media_id}", + f"/_matrix/client/v1/media/download/{server_and_media_id}", shorthand=False, access_token=user_token, ) diff --git a/tests/rest/media/test_domain_blocking.py b/tests/rest/media/test_domain_blocking.py index 72205c6bb3b..49d81f4b286 100644 --- a/tests/rest/media/test_domain_blocking.py +++ b/tests/rest/media/test_domain_blocking.py @@ -91,7 +91,8 @@ def test_cannot_download_blocked_media(self) -> None: { # Disable downloads from a domain we won't be requesting downloads from. # This proves we haven't broken anything. - "prevent_media_downloads_from": ["not-listed.com"] + "prevent_media_downloads_from": ["not-listed.com"], + "enable_authenticated_media": False, } ) def test_remote_media_normally_unblocked(self) -> None: @@ -132,6 +133,7 @@ def test_cannot_download_blocked_media_thumbnail(self) -> None: # This proves we haven't broken anything. "prevent_media_downloads_from": ["not-listed.com"], "dynamic_thumbnails": True, + "enable_authenticated_media": False, } ) def test_remote_media_thumbnail_normally_unblocked(self) -> None: diff --git a/tests/rest/media/test_url_preview.py b/tests/rest/media/test_url_preview.py index a96f0e7fca2..103d7662d9d 100644 --- a/tests/rest/media/test_url_preview.py +++ b/tests/rest/media/test_url_preview.py @@ -42,6 +42,7 @@ from tests import unittest from tests.server import FakeTransport from tests.test_utils import SMALL_PNG +from tests.unittest import override_config try: import lxml @@ -1259,6 +1260,7 @@ def _download_image(self) -> Tuple[str, str]: self.assertIsNone(_port) return host, media_id + @override_config({"enable_authenticated_media": False}) def test_storage_providers_exclude_files(self) -> None: """Test that files are not stored in or fetched from storage providers.""" host, media_id = self._download_image() @@ -1301,6 +1303,7 @@ def test_storage_providers_exclude_files(self) -> None: "URL cache file was unexpectedly retrieved from a storage provider", ) + @override_config({"enable_authenticated_media": False}) def test_storage_providers_exclude_thumbnails(self) -> None: """Test that thumbnails are not stored in or fetched from storage providers.""" host, media_id = self._download_image()