Enable communication with "authorized_fetch" Mastodon servers

bookwyrm/activitypub/base_activity.py

@@ -2,11 +2,16 @@
 from dataclasses import dataclass, fields, MISSING
 from json import JSONEncoder
 import logging
+import requests
 
 from django.apps import apps
 from django.db import IntegrityError, transaction
+from django.utils.http import http_date
 
+from bookwyrm import models
 from bookwyrm.connectors import ConnectorException, get_data
+from bookwyrm.signatures import make_signature
+from bookwyrm.settings import DOMAIN, INSTANCE_ACTOR_USERNAME
 from bookwyrm.tasks import app, MEDIUM
 
 logger = logging.getLogger(__name__)
@@ -246,10 +251,10 @@ def set_related_field(
 
 def get_model_from_type(activity_type):
     """given the activity, what type of model"""
-    models = apps.get_models()
+    activity_models = apps.get_models()
     model = [
         m
-        for m in models
+        for m in activity_models
         if hasattr(m, "activity_serializer")
         and hasattr(m.activity_serializer, "type")
         and m.activity_serializer.type == activity_type
@@ -275,10 +280,16 @@ def resolve_remote_id(
     # load the data and create the object
     try:
         data = get_data(remote_id)
-    except ConnectorException:
+    except ConnectionError:
         logger.info("Could not connect to host for remote_id: %s", remote_id)
         return None
-
+    except requests.HTTPError as e:
+        if (e.response is not None) and e.response.status_code == 401:
+            # This most likely means it's a mastodon with secure fetch enabled.
+            data = get_activitypub_data(remote_id)
+        else:
+            logger.info("Could not connect to host for remote_id: %s", remote_id)
+            return None
     # determine the model implicitly, if not provided
     # or if it's a model with subclasses like Status, check again
     if not model or hasattr(model.objects, "select_subclasses"):
@@ -297,6 +308,51 @@ def resolve_remote_id(
     return item.to_model(model=model, instance=result, save=save)
 
 
+def get_representative():
+    """Get or create an actor representing the instance
+    to sign requests to 'secure mastodon' servers"""
+    username = f"{INSTANCE_ACTOR_USERNAME}@{DOMAIN}"
+    email = "bookwyrm@localhost"
+    try:
+        user = models.User.objects.get(username=username)
+    except models.User.DoesNotExist:
+        user = models.User.objects.create_user(
+            username=username,
+            email=email,
+            local=True,
+            localname=INSTANCE_ACTOR_USERNAME,
+        )
+    return user
+
+
+def get_activitypub_data(url):
+    """wrapper for request.get"""
+    now = http_date()
+    sender = get_representative()
+    if not sender.key_pair.private_key:
+        # this shouldn't happen. it would be bad if it happened.
+        raise ValueError("No private key found for sender")
+    try:
+        resp = requests.get(
+            url,
+            headers={
+                "Accept": "application/json; charset=utf-8",
+                "Date": now,
+                "Signature": make_signature("get", sender, url, now),
+            },
+        )
+    except requests.RequestException:
+        raise ConnectorException()
+    if not resp.ok:
+        resp.raise_for_status()
+    try:
+        data = resp.json()
+    except ValueError:
+        raise ConnectorException()
+
+    return data
+
+
 @dataclass(init=False)
 class Link(ActivityObject):
     """for tagging a book in a status"""

bookwyrm/connectors/abstract_connector.py

@@ -244,7 +244,11 @@ def get_data(url, params=None, timeout=settings.QUERY_TIMEOUT):
         raise ConnectorException(err)
 
     if not resp.ok:
-        raise ConnectorException()
+        if resp.status_code == 401:
+            # this is probably an AUTHORIZED_FETCH issue
+            resp.raise_for_status()
+        else:
+            raise ConnectorException()
     try:
         data = resp.json()
     except ValueError as err:

bookwyrm/models/activitypub_mixin.py

@@ -543,7 +543,7 @@ async def sign_and_send(
     headers = {
         "Date": now,
         "Digest": digest,
-        "Signature": make_signature(sender, destination, now, digest),
+        "Signature": make_signature("post", sender, destination, now, digest),
         "Content-Type": "application/activity+json; charset=utf-8",
         "User-Agent": USER_AGENT,
     }

bookwyrm/settings.py

@@ -370,3 +370,9 @@
 HTTP_X_FORWARDED_PROTO = env.bool("SECURE_PROXY_SSL_HEADER", False)
 if HTTP_X_FORWARDED_PROTO:
     SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https")
+
+# Instance Actor for signing GET requests to "secure mode"
+# Mastodon servers.
+# Do not change this setting unless you already have an existing
+# user with the same username - in which case you should change it!
+INSTANCE_ACTOR_USERNAME = "bookwyrm.instance.actor"

bookwyrm/signatures.py

@@ -22,22 +22,26 @@ def create_key_pair():
     return private_key, public_key
 
 
-def make_signature(sender, destination, date, digest):
+def make_signature(method, sender, destination, date, digest=None):
     """uses a private key to sign an outgoing message"""
     inbox_parts = urlparse(destination)
     signature_headers = [
-        f"(request-target): post {inbox_parts.path}",
+        f"(request-target): {method} {inbox_parts.path}",
         f"host: {inbox_parts.netloc}",
         f"date: {date}",
-        f"digest: {digest}",
     ]
+    headers = "(request-target) host date"
+    if digest is not None:
+        signature_headers.append(f"digest: {digest}")
+        headers = "(request-target) host date digest"
+
     message_to_sign = "\n".join(signature_headers)
     signer = pkcs1_15.new(RSA.import_key(sender.key_pair.private_key))
     signed_message = signer.sign(SHA256.new(message_to_sign.encode("utf8")))
     signature = {
         "keyId": f"{sender.remote_id}#main-key",
         "algorithm": "rsa-sha256",
-        "headers": "(request-target) host date digest",
+        "headers": headers,
         "signature": b64encode(signed_message).decode("utf8"),
     }
     return ",".join(f'{k}="{v}"' for (k, v) in signature.items())

bookwyrm/tests/activitypub/test_base_activity.py

@@ -14,6 +14,7 @@
     ActivityObject,
     resolve_remote_id,
     set_related_field,
+    get_representative,
 )
 from bookwyrm.activitypub import ActivitySerializerError
 from bookwyrm import models
@@ -52,6 +53,11 @@ def setUp(self):
         image.save(output, format=image.format)
         self.image_data = output.getvalue()
 
+    def test_get_representative_not_existing(self, *_):
+        """test that an instance representative actor is created if it does not exist"""
+        representative = get_representative()
+        self.assertIsInstance(representative, models.User)
+
     def test_init(self, *_):
         """simple successfuly init"""
         instance = ActivityObject(id="a", type="b")

bookwyrm/tests/test_signing.py

@@ -86,7 +86,9 @@ def send_test_request(  # pylint: disable=too-many-arguments
         now = date or http_date()
         data = json.dumps(get_follow_activity(sender, self.rat))
         digest = digest or make_digest(data)
-        signature = make_signature(signer or sender, self.rat.inbox, now, digest)
+        signature = make_signature(
+            "post", signer or sender, self.rat.inbox, now, digest
+        )
         with patch("bookwyrm.views.inbox.activity_task.apply_async"):
             with patch("bookwyrm.models.user.set_remote_server.delay"):
                 return self.send(signature, now, send_data or data, digest)