MSC2246 async uploads

changelog.d/12484.feature

@@ -0,0 +1 @@
+Experimental support for asynchronous uploads as defined by [MSC2246](https://github.com/matrix-org/matrix-spec-proposals/pull/2246). Contributed by @sumnerevans at Beeper.

docs/sample_config.yaml

@@ -964,6 +964,10 @@ log_config: "CONFDIR/SERVERNAME.log.config"
 #rc_third_party_invite:
 #  per_second: 0.2
 #  burst_count: 10
+#
+#rc_media_create:
+#  per_second: 10
+#  burst_count: 50
 
 # Ratelimiting settings for incoming federation
 #
@@ -1034,6 +1038,11 @@ media_store_path: "DATADIR/media_store"
 #
 #max_image_pixels: 32M
 
+# How long to wait before expiring created media IDs when MSC2246 support is
+# enabled.
+#
+#unused_expiration_time: 1m
+
 # Whether to generate new thumbnails on the fly to precisely match
 # the resolution requested by the client. If true then whenever
 # a new resolution is requested by the client the server will

synapse/api/urls.py

@@ -29,9 +29,7 @@
 FEDERATION_UNSTABLE_PREFIX = FEDERATION_PREFIX + "/unstable"
 STATIC_PREFIX = "/_matrix/static"
 SERVER_KEY_V2_PREFIX = "/_matrix/key/v2"
-MEDIA_R0_PREFIX = "/_matrix/media/r0"
-MEDIA_V3_PREFIX = "/_matrix/media/v3"
-LEGACY_MEDIA_PREFIX = "/_matrix/media/v1"
+MEDIA_PREFIX = "/_matrix/media"
 
 
 class ConsentURIBuilder:

synapse/app/generic_worker.py

@@ -27,9 +27,7 @@
 from synapse.api.urls import (
     CLIENT_API_PREFIX,
     FEDERATION_PREFIX,
-    LEGACY_MEDIA_PREFIX,
-    MEDIA_R0_PREFIX,
-    MEDIA_V3_PREFIX,
+    MEDIA_PREFIX,
     SERVER_KEY_V2_PREFIX,
 )
 from synapse.app import _base
@@ -338,9 +336,7 @@ def _listen_http(self, listener_config: ListenerConfig) -> None:
 
                         resources.update(
                             {
-                                MEDIA_R0_PREFIX: media_repo,
-                                MEDIA_V3_PREFIX: media_repo,
-                                LEGACY_MEDIA_PREFIX: media_repo,
+                                MEDIA_PREFIX: media_repo,
                                 "/_synapse/admin": admin_resource,
                             }
                         )

synapse/app/homeserver.py

@@ -30,9 +30,7 @@
 from synapse.api.urls import (
     CLIENT_API_PREFIX,
     FEDERATION_PREFIX,
-    LEGACY_MEDIA_PREFIX,
-    MEDIA_R0_PREFIX,
-    MEDIA_V3_PREFIX,
+    MEDIA_PREFIX,
     SERVER_KEY_V2_PREFIX,
     STATIC_PREFIX,
 )
@@ -245,13 +243,7 @@ def _configure_named_resource(
         if name in ["media", "federation", "client"]:
             if self.config.server.enable_media_repo:
                 media_repo = self.get_media_repository_resource()
-                resources.update(
-                    {
-                        MEDIA_R0_PREFIX: media_repo,
-                        MEDIA_V3_PREFIX: media_repo,
-                        LEGACY_MEDIA_PREFIX: media_repo,
-                    }
-                )
+                resources[MEDIA_PREFIX] = media_repo
             elif name == "media":
                 raise ConfigError(
                     "'media' resource conflicts with enable_media_repo=False"

synapse/config/experimental.py

@@ -84,3 +84,6 @@ def read_config(self, config: JsonDict, **kwargs: Any) -> None:
 
         # MSC3772: A push rule for mutual relations.
         self.msc3772_enabled: bool = experimental.get("msc3772_enabled", False)
+
+        # MSC2246 (async media uploads)
+        self.msc2246_enabled: bool = experimental.get("msc2246_enabled", False)

synapse/config/ratelimiting.py

@@ -144,6 +144,12 @@ def read_config(self, config: JsonDict, **kwargs: Any) -> None:
             },
         )
 
+        # Ratelimit create media requests:
+        self.rc_media_create = RateLimitConfig(
+            config.get("rc_media_create", {}),
+            defaults={"per_second": 10, "burst_count": 50},
+        )
+
     def generate_config_section(self, **kwargs: Any) -> str:
         return """\
         ## Ratelimiting ##
@@ -234,6 +240,10 @@ def generate_config_section(self, **kwargs: Any) -> str:
         #rc_third_party_invite:
         #  per_second: 0.2
         #  burst_count: 10
+        #
+        #rc_media_create:
+        #  per_second: 10
+        #  burst_count: 50
 
         # Ratelimiting settings for incoming federation
         #

synapse/config/repository.py

@@ -117,6 +117,10 @@ def read_config(self, config: JsonDict, **kwargs: Any) -> None:
         self.max_image_pixels = self.parse_size(config.get("max_image_pixels", "32M"))
         self.max_spider_size = self.parse_size(config.get("max_spider_size", "10M"))
 
+        self.unused_expiration_time = self.parse_duration(
+            config.get("unused_expiration_time", "1m")
+        )
+
         self.media_store_path = self.ensure_directory(
             config.get("media_store_path", "media_store")
         )
@@ -292,6 +296,11 @@ def generate_config_section(self, data_dir_path: str, **kwargs: Any) -> str:
         #
         #max_image_pixels: 32M
 
+        # How long to wait before expiring created media IDs when MSC2246 support is
+        # enabled.
+        #
+        #unused_expiration_time: 1m
+
         # Whether to generate new thumbnails on the fly to precisely match
         # the resolution requested by the client. If true then whenever
         # a new resolution is requested by the client the server will

synapse/rest/media/v1/_base.py

@@ -49,6 +49,8 @@
     "text/xml",
 ]
 
+DEFAULT_MSC2246_DELAY = 20_000
+
 
 def parse_media_id(request: Request) -> Tuple[str, str, Optional[str]]:
     """Parses the server name, media ID and optional file name from the request URI

synapse/rest/media/v1/create_resource.py

@@ -0,0 +1,84 @@
+# Copyright 2014-2016 OpenMarket Ltd
+# Copyright 2020-2021 The Matrix.org Foundation C.I.C.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+from typing import TYPE_CHECKING
+
+from synapse.api.errors import LimitExceededError
+from synapse.api.ratelimiting import Ratelimiter
+from synapse.http.server import DirectServeJsonResource, respond_with_json
+from synapse.http.site import SynapseRequest
+
+if TYPE_CHECKING:
+    from synapse.rest.media.v1.media_repository import MediaRepository
+    from synapse.server import HomeServer
+
+
+logger = logging.getLogger(__name__)
+
+
+class CreateResource(DirectServeJsonResource):
+    isLeaf = True
+
+    def __init__(self, hs: "HomeServer", media_repo: "MediaRepository"):
+        super().__init__()
+
+        self.media_repo = media_repo
+        self.clock = hs.get_clock()
+        self.auth = hs.get_auth()
+
+        # A rate limiter for creating new media IDs.
+        self._create_media_rate_limiter = Ratelimiter(
+            store=hs.get_datastores().main,
+            clock=self.clock,
+            rate_hz=hs.config.ratelimiting.rc_media_create.per_second,
+            burst_count=hs.config.ratelimiting.rc_media_create.burst_count,
+        )
+
+    async def _async_render_OPTIONS(self, request: SynapseRequest) -> None:
+        respond_with_json(request, 200, {}, send_cors=True)
+
+    async def _async_render_POST(self, request: SynapseRequest) -> None:
+        requester = await self.auth.get_user_by_req(request)
+
+        # If the create media requests for the user are over the limit, drop
+        # them.
+        allowed, time_allowed = await self._create_media_rate_limiter.can_do_action(
+            requester
+        )
+        if not allowed:
+            time_now_s = self.clock.time()
+            raise LimitExceededError(
+                retry_after_ms=int(1000 * (time_allowed - time_now_s))
+            )
+
+        content_uri, unused_expires_at = await self.media_repo.create_media_id(
+            requester.user
+        )
+
+        logger.info(
+            "Created Media URI %r that if unused will expire at %d",
+            content_uri,
+            unused_expires_at,
+        )
+        respond_with_json(
+            request,
+            200,
+            {
+                "content_uri": content_uri,
+                "unused_expires_at": unused_expires_at,
+            },
+            send_cors=True,
+        )

synapse/rest/media/v1/download_resource.py

@@ -16,10 +16,10 @@
 from typing import TYPE_CHECKING
 
 from synapse.http.server import DirectServeJsonResource, set_cors_headers
-from synapse.http.servlet import parse_boolean
+from synapse.http.servlet import parse_boolean, parse_integer
 from synapse.http.site import SynapseRequest
 
-from ._base import parse_media_id, respond_404
+from ._base import DEFAULT_MSC2246_DELAY, parse_media_id, respond_404
 
 if TYPE_CHECKING:
     from synapse.rest.media.v1.media_repository import MediaRepository
@@ -35,6 +35,7 @@ def __init__(self, hs: "HomeServer", media_repo: "MediaRepository"):
         super().__init__()
         self.media_repo = media_repo
         self.server_name = hs.hostname
+        self.enable_msc2246 = hs.config.experimental.msc2246_enabled
 
     async def _async_render_GET(self, request: SynapseRequest) -> None:
         set_cors_headers(request)
@@ -50,13 +51,14 @@ async def _async_render_GET(self, request: SynapseRequest) -> None:
         )
         # Limited non-standard form of CSP for IE11
         request.setHeader(b"X-Content-Security-Policy", b"sandbox;")
-        request.setHeader(
-            b"Referrer-Policy",
-            b"no-referrer",
-        )
+        request.setHeader(b"Referrer-Policy", b"no-referrer")
         server_name, media_id, name = parse_media_id(request)
+        max_stall_ms = parse_integer(
+            request, "fi.mau.msc2246.max_stall_ms", default=DEFAULT_MSC2246_DELAY
+        )
+
         if server_name == self.server_name:
-            await self.media_repo.get_local_media(request, media_id, name)
+            await self.media_repo.get_local_media(request, media_id, name, max_stall_ms)
         else:
             allow_remote = parse_boolean(request, "allow_remote", default=True)
             if not allow_remote:
@@ -68,4 +70,6 @@ async def _async_render_GET(self, request: SynapseRequest) -> None:
                 respond_404(request)
                 return
 
-            await self.media_repo.get_remote_media(request, server_name, media_id, name)
+            await self.media_repo.get_remote_media(
+                request, server_name, media_id, name, max_stall_ms
+            )