Merge branch 'async-subtree-translation'

docs/how-to/configure-background-tasks.md

@@ -0,0 +1,53 @@
+# Configuring background tasks
+
+In Wagtail Localize, it's possible to perform bulk actions such as submitting an entire page tree for translation or create a new locale by copying all the content from an existing one.
+
+While these are powerful features, they can be quite slow on large sites (where you might be submitting >100 pages at a time) and this can even cause timeouts.
+
+To mitigate this, Wagtail Localize provides a mechanism to allow you to configure these tasks to be run in a background worker instead.
+
+Currently, Wagtail Localize supports [Django RQ](https://github.com/rq/django-rq) out of the box, and you can implement support for others as documented below
+
+## Configuring Django RQ
+
+Wagtail Localize has built-in support for Django RQ. So if you already have Django RQ installed, you can configure it with the following setting:
+
+```python
+WAGTAILLOCALIZE_JOBS = {
+    "BACKEND": "wagtail_localize.tasks.DjangoRQJobBackend",
+    "OPTIONS": {"QUEUE": "default"},
+}
+```
+
+The `OPTIONS` => `QUEUE` key configures the Django RQ queue to push tasks to.
+
+## Configuring a different queueing system
+
+To configure any other queueing system, create a subclass of `wagtail_localize.tasks.BaseJobBackend` somewhere in your project and override the `__init__` and `enqueue` methods:
+
+```python
+from wagtail_localize.tasks import BaseJobBacked
+
+
+class MyJobBackend(BaseJobBackend):
+    def __init__(self, options):
+        # Any set up code goes here. Note that the 'options' parameter contains the value of WAGTAILLOCALIZE_JOBS["OPTIONS"]
+        pass
+
+    def enqueue(self, func, args, kwargs):
+        # func is a function object to call
+        # args is a list of positional arguments to pass into the function when it's called
+        # kwargs is is a dict of keyword arguments to pass into the function when it's called
+        pass
+```
+
+When you've implemented that class, hook it in to Wagtail Localize using the `WAGTAILLOCALIZE_JOBS` setting:
+
+```python
+WAGTAILLOCALIZE_JOBS = {
+    "BACKEND": "python.path.to.MyJobBackend",
+    "OPTIONS": {
+        # Any options can go here
+    },
+}
+```

mkdocs.yml

@@ -50,6 +50,7 @@ nav:
       - Installation guide: how-to/installation.md
       - Configuring translatable fields: how-to/field-configuration.md
       - Translatable ModelAdmin: how-to/modeladmin.md
+      - Configuring background tasks: how-to/configure-background-tasks.md
       - Integrations:
           - Pontoon: how-to/integrations/pontoon.md
           - Machine Translation: how-to/integrations/machine-translation.md

setup.py

@@ -44,9 +44,18 @@
         "Framework :: Wagtail",
         "Framework :: Wagtail :: 2",
     ],
-    install_requires=["Django>=2.2,<4.1", "Wagtail>=2.11,<2.17", "polib>=1.1,<2.0"],
+    install_requires=[
+        "Django>=2.2,<4.1",
+        "Wagtail>=2.11,<2.17",
+        "polib>=1.1,<2.0",
+        "typing_extensions>=4.0",
+    ],
     extras_require={
-        "testing": ["dj-database-url==0.5.0", "freezegun==1.1.0"],
+        "testing": [
+            "dj-database-url==0.5.0",
+            "freezegun==1.1.0",
+            "django-rq>=2.5,<3.0",
+        ],
         "documentation": [
             "mkdocs==1.1.2",
             "mkdocs-material==6.2.8",

wagtail_localize/models.py

@@ -61,6 +61,7 @@
 from .segments.extract import extract_segments
 from .segments.ingest import ingest_segments
 from .strings import StringValue, validate_translation_links
+from .tasks import background
 
 
 if WAGTAIL_VERSION >= (2, 16):
@@ -2245,7 +2246,11 @@ class LocaleSynchronization(models.Model):
     def sync_trees(self, *, page_index=None):
         from .synctree import synchronize_tree
 
-        synchronize_tree(self.sync_from, self.locale, page_index=page_index)
+        background.enqueue(
+            synchronize_tree,
+            args=[self.sync_from, self.locale],
+            kwargs={"page_index": page_index},
+        )
 
 
 @receiver(post_save, sender=LocaleSynchronization)

wagtail_localize/operations.py

@@ -0,0 +1,117 @@
+from collections import defaultdict
+
+from django.conf import settings
+from django.core.exceptions import ValidationError
+from django.db import transaction
+from wagtail.core.models import Page
+
+from wagtail_localize.models import Translation, TranslationSource
+
+
+class TranslationCreator:
+    """
+    A class that provides a create_translations method.
+
+    Call create_translations for each object you want to translate and this will submit
+    that object and any dependencies as well.
+
+    This class will track the objects that have already submitted so an object doesn't
+    get submitted twice.
+    """
+
+    def __init__(self, user, target_locales):
+        self.user = user
+        self.target_locales = target_locales
+        self.seen_objects = set()
+        self.mappings = defaultdict(list)
+
+    def create_translations(self, instance, include_related_objects=True):
+        if isinstance(instance, Page):
+            instance = instance.specific
+
+        if instance.translation_key in self.seen_objects:
+            return
+        self.seen_objects.add(instance.translation_key)
+
+        source, created = TranslationSource.get_or_create_from_instance(instance)
+
+        # Add related objects
+        # Must be before translation records or those translation records won't be able to create
+        # the objects because the dependencies haven't been created
+        if include_related_objects:
+            for related_object_segment in source.relatedobjectsegment_set.all():
+                related_instance = related_object_segment.object.get_instance(
+                    instance.locale
+                )
+
+                # Limit to one level of related objects, since this could potentially pull in a lot of stuff
+                self.create_translations(
+                    related_instance, include_related_objects=False
+                )
+
+        # Support disabling the out of the box translation mode.
+        # The value set on the model takes precendence over the global setting.
+        if hasattr(instance, "localize_default_translation_mode"):
+            translation_mode = instance.localize_default_translation_mode
+        else:
+            translation_mode = getattr(
+                settings, "WAGTAIL_LOCALIZE_DEFAULT_TRANSLATION_MODE", "synced"
+            )
+        translation_enabled = translation_mode == "synced"
+
+        # Set up translation records
+        for target_locale in self.target_locales:
+            # Create translation if it doesn't exist yet, re-enable if translation was disabled
+            # Note that the form won't show this locale as an option if the translation existed
+            # in this langauge, so this shouldn't overwrite any unmanaged translations.
+            translation, created = Translation.objects.update_or_create(
+                source=source,
+                target_locale=target_locale,
+                defaults={"enabled": translation_enabled},
+            )
+
+            self.mappings[source].append(translation)
+
+            try:
+                translation.save_target(user=self.user)
+            except ValidationError:
+                pass
+
+
+@transaction.atomic
+def translate_object(instance, locales, components=None, user=None):
+    """
+    Translates the given object into the given locales.
+    """
+    translator = TranslationCreator(user, locales)
+    translator.create_translations(instance)
+
+    if components is not None:
+        components.save(translator, sources_and_translations=translator.mappings)
+
+
+@transaction.atomic
+def translate_page_subtree(page_id, locales, components, user):
+    """
+    Translates the given page's subtree into the given locales.
+
+    Note that the page itself must already be translated.
+
+    Note: Page must be passed by ID since this function may be called with an async worker and pages can't be reliably pickled.
+          See: https://github.com/wagtail/wagtail/pull/5998
+    """
+    page = Page.objects.get(id=page_id)
+
+    translator = TranslationCreator(user, locales)
+
+    def _walk(current_page):
+        for child_page in current_page.get_children():
+            translator.create_translations(child_page)
+
+            if child_page.numchild:
+                _walk(child_page)
+
+    _walk(page)
+
+    if components is not None:
+        components.save(translator, sources_and_translations=translator.mappings)

wagtail_localize/tasks.py

@@ -0,0 +1,47 @@
+# This file contains a very lightweight implementation of RFC 72: Background workers (https://github.com/wagtail/rfcs/pull/72)
+# This is only to be used by Wagtail Localize and will be replaced with the full Wagtail implementation later
+
+from typing import Any, Callable
+
+from django.conf import settings
+from django.utils.module_loading import import_string
+from typing_extensions import ParamSpec
+
+
+P = ParamSpec("P")
+
+
+class BaseJobBackend:
+    def __init__(self, options):
+        pass
+
+    def enqueue(self, func: Callable[P, Any], args: P.args, kwargs: P.kwargs):
+        raise NotImplementedError()
+
+
+class ImmediateBackend(BaseJobBackend):
+    def enqueue(self, func: Callable[P, Any], args: P.args, kwargs: P.kwargs):
+        func(*args, **kwargs)
+
+
+class DjangoRQJobBackend(BaseJobBackend):
+    def __init__(self, options):
+        import django_rq
+
+        self.queue = django_rq.get_queue(options.get("QUEUE", "default"))
+
+    def enqueue(self, func: Callable[P, Any], args: P.args, kwargs: P.kwargs):
+        self.queue.enqueue(func, *args, **kwargs)
+
+
+def get_backend():
+    config = getattr(
+        settings,
+        "WAGTAILLOCALIZE_JOBS",
+        {"BACKEND": "wagtail_localize.tasks.ImmediateBackend"},
+    )
+    backend_class = import_string(config["BACKEND"])
+    return backend_class(config.get("OPTIONS", {}))
+
+
+background: BaseJobBackend = get_backend()