Merge pull request #1268 from kollivier/node_details_async

Switch get_node_details to always use cache when available...

contentcuration/contentcuration/models.py

@@ -3,10 +3,13 @@
 import json
 import logging
 import os
+import pytz
 import urlparse
 import uuid
 import warnings
 
+from datetime import datetime
+
 from django.conf import settings
 from django.contrib.auth.base_user import AbstractBaseUser
 from django.contrib.auth.base_user import BaseUserManager
@@ -22,6 +25,7 @@
 from django.db import connection
 from django.db import IntegrityError
 from django.db import models
+from django.db.models import Count
 from django.db.models import Max
 from django.db.models import Q
 from django.db.models import Sum
@@ -42,6 +46,7 @@
 from pg_utils import DistinctSum
 
 from contentcuration.statistics import record_channel_stats
+from contentcuration.utils.parser import load_json_string
 
 EDIT_ACCESS = "edit"
 VIEW_ACCESS = "view"
@@ -958,6 +963,19 @@ def get_channel(self):
         except (ObjectDoesNotExist, MultipleObjectsReturned, AttributeError):
             return None
 
+    def get_thumbnail(self):
+        # Problems with json.loads, so use ast.literal_eval to get dict
+        if self.thumbnail_encoding:
+            thumbnail_data = load_json_string(self.thumbnail_encoding)
+            if thumbnail_data.get("base64"):
+                return thumbnail_data["base64"]
+
+        thumbnail = self.files.filter(preset__thumbnail=True).first()
+        if thumbnail:
+            return generate_storage_url(str(thumbnail))
+
+        return "/".join([settings.STATIC_URL.rstrip("/"), "img", "{}_placeholder.png".format(self.kind_id)])
+
     @classmethod
     def get_nodes_with_title(cls, title, limit_to_children_of=None):
         """
@@ -970,6 +988,109 @@ def get_nodes_with_title(cls, title, limit_to_children_of=None):
         else:
             return cls.objects.filter(title=title)
 
+    def get_details(self):
+        """
+        Returns information about the node and its children, including total size, languages, files, etc.
+
+        :return: A dictionary with detailed statistics and information about the node.
+        """
+        descendants = self.get_descendants().prefetch_related('children', 'files', 'tags') \
+            .select_related('license', 'language')
+        channel = self.get_channel()
+
+        # Get resources
+        resources = descendants.exclude(kind=content_kinds.TOPIC)
+
+        # Get all copyright holders, authors, aggregators, and providers and split into lists
+        creators = resources.values_list('copyright_holder', 'author', 'aggregator', 'provider')
+        split_lst = zip(*creators)
+        copyright_holders = filter(bool, set(split_lst[0])) if len(split_lst) > 0 else []
+        authors = filter(bool, set(split_lst[1])) if len(split_lst) > 1 else []
+        aggregators = filter(bool, set(split_lst[2])) if len(split_lst) > 2 else []
+        providers = filter(bool, set(split_lst[3])) if len(split_lst) > 3 else []
+
+        # Get sample pathway by getting longest path
+        # Using resources.aggregate adds a lot of time, use values that have already been fetched
+        max_level = max(resources.values_list('level', flat=True).distinct() or [0])
+        deepest_node = resources.filter(level=max_level).first()
+        pathway = list(deepest_node.get_ancestors()
+                       .exclude(parent=None)
+                       .values('title', 'node_id', 'kind_id')
+                       ) if deepest_node else []
+        sample_nodes = [
+            {
+                "node_id": n.node_id,
+                "title": n.title,
+                "description": n.description,
+                "thumbnail": n.get_thumbnail(),
+            } for n in deepest_node.get_siblings(include_self=True)[0:4]
+        ] if deepest_node else []
+
+        # Get list of channels nodes were originally imported from (omitting the current channel)
+        channel_id = channel and channel.id
+        originals = resources.values("original_channel_id") \
+            .annotate(count=Count("original_channel_id")) \
+            .order_by("original_channel_id")
+        originals = {c['original_channel_id']: c['count'] for c in originals}
+        original_channels = Channel.objects.exclude(pk=channel_id) \
+            .filter(pk__in=[k for k, v in originals.items()], deleted=False)
+        original_channels = [{
+            "id": c.id,
+            "name": "{}{}".format(c.name, _(" (Original)") if channel_id == c.id else ""),
+            "thumbnail": c.get_thumbnail(),
+            "count": originals[c.id]
+        } for c in original_channels]
+
+        # Get tags from channel
+        tags = list(ContentTag.objects.filter(tagged_content__pk__in=descendants.values_list('pk', flat=True))
+                    .values('tag_name')
+                    .annotate(count=Count('tag_name'))
+                    .order_by('tag_name'))
+
+        # Get resource variables
+        resource_count = resources.count() or 0
+        resource_size = resources.values('files__checksum', 'files__file_size').distinct().aggregate(
+            resource_size=Sum('files__file_size'))['resource_size'] or 0
+
+        languages = list(set(descendants.exclude(language=None).values_list('language__native_name', flat=True)))
+        accessible_languages = resources.filter(files__preset_id=format_presets.VIDEO_SUBTITLE) \
+            .values_list('files__language_id', flat=True)
+        accessible_languages = list(
+            Language.objects.filter(id__in=accessible_languages).distinct().values_list('native_name', flat=True))
+
+        licenses = list(set(resources.exclude(license=None).values_list('license__license_name', flat=True)))
+        kind_count = list(resources.values('kind_id').annotate(count=Count('kind_id')).order_by('kind_id'))
+
+        # Add "For Educators" booleans
+        for_educators = {
+            "coach_content": resources.filter(role_visibility=roles.COACH).exists(),
+            "exercises": resources.filter(kind_id=content_kinds.EXERCISE).exists(),
+        }
+
+        # Serialize data
+        data = {
+            "last_update": pytz.utc.localize(datetime.now()).strftime(settings.DATE_TIME_FORMAT),
+            "resource_count": resource_count,
+            "resource_size": resource_size,
+            "includes": for_educators,
+            "kind_count": kind_count,
+            "languages": languages,
+            "accessible_languages": accessible_languages,
+            "licenses": licenses,
+            "tags": tags,
+            "copyright_holders": copyright_holders,
+            "authors": authors,
+            "aggregators": aggregators,
+            "providers": providers,
+            "sample_pathway": pathway,
+            "original_channels": original_channels,
+            "sample_nodes": sample_nodes,
+        }
+
+        # Set cache with latest data
+        cache.set("details_{}".format(self.node_id), json.dumps(data), None)
+        return data
+
     def save(self, *args, **kwargs):  # noqa: C901
 
         channel_id = None

contentcuration/contentcuration/settings.py

@@ -280,6 +280,9 @@ def ugettext(s): return s
 REGISTRATION_OPEN = True
 SITE_ID = 1
 
+# Used for serializing datetime objects.
+DATE_TIME_FORMAT = "%Y-%m-%d %H:%M:%S"
+
 # EMAIL_HOST = 'localhost'
 # EMAIL_PORT = 8000
 # EMAIL_HOST_USER = ''

contentcuration/contentcuration/tasks.py

@@ -85,6 +85,12 @@ def deletetree_task(tree_id):
     ContentNode.objects.filter(tree_id=tree_id).delete()
 
 
+@task(name='getnodedetails_task')
+def getnodedetails_task(node_id):
+    node = ContentNode.objects.get(pk=node_id)
+    return node.get_details()
+
+
 type_mapping = {}
 if settings.RUNNING_TESTS:
     type_mapping.update({

contentcuration/contentcuration/tests/test_channel_model.py

@@ -7,10 +7,12 @@
 
 from .base import BaseAPITestCase
 from .base import StudioTestCase
+from .testdata import base64encoding
 from .testdata import channel
 from .testdata import node
 from contentcuration.models import Channel
 from contentcuration.models import ChannelSet
+from contentcuration.models import generate_storage_url
 from contentcuration.models import SecretToken
 
 
@@ -330,3 +332,21 @@ def test_change_public(self):
 
             c.main_tree.refresh_from_db()
             self.assertFalse(c.main_tree.changed)
+
+
+class ChannelGettersTestCase(BaseAPITestCase):
+    def test_get_channel_thumbnail_default(self):
+        default_thumbnail = '/static/img/kolibri_placeholder.png'
+        thumbnail = self.channel.get_thumbnail()
+        assert thumbnail == default_thumbnail
+
+    def test_get_channel_thumbnail_base64(self):
+        self.channel.thumbnail_encoding = {"base64": base64encoding()}
+
+        assert self.channel.get_thumbnail() == base64encoding()
+
+    def test_get_channel_thumbnail_file(self):
+        thumbnail_url = '/path/to/thumbnail.png'
+        self.channel.thumbnail = thumbnail_url
+
+        assert self.channel.get_thumbnail() == generate_storage_url(thumbnail_url)

contentcuration/contentcuration/tests/test_contentnodes.py

@@ -1,15 +1,20 @@
 import json
 
 import testdata
-from testdata import create_temp_file
+from .testdata import create_temp_file
 from base import BaseAPITestCase
 from base import BaseTestCase
+
+from django.conf import settings
 from django.core.urlresolvers import reverse_lazy
 
 from contentcuration.models import Channel
 from contentcuration.models import ContentKind
 from contentcuration.models import ContentNode
+from contentcuration.models import FormatPreset
+from contentcuration.models import generate_storage_url
 from contentcuration.models import Language
+from contentcuration.utils.files import create_thumbnail_from_base64
 from contentcuration.views import nodes
 
 
@@ -37,6 +42,49 @@ def _check_nodes(parent, title=None, original_channel_id=None, source_channel_id
         _check_nodes(node, title, original_channel_id, source_channel_id, channel)
 
 
+class NodeGettersTestCase(BaseTestCase):
+    def setUp(self):
+        super(NodeGettersTestCase, self).setUp()
+
+        self.channel = testdata.channel()
+        self.topic, _created = ContentKind.objects.get_or_create(kind="Topic")
+        self.thumbnail_data = "allyourbase64arebelongtous"
+
+    def test_get_node_thumbnail_default(self):
+        new_node = ContentNode.objects.create(title="Heyo!", parent=self.channel.main_tree, kind=self.topic)
+
+        default_thumbnail = "/".join([settings.STATIC_URL.rstrip("/"), "img", "{}_placeholder.png".format(new_node.kind_id)])
+        thumbnail = new_node.get_thumbnail()
+        assert thumbnail == default_thumbnail
+
+    def test_get_node_thumbnail_base64(self):
+        new_node = ContentNode.objects.create(title="Heyo!", parent=self.channel.main_tree, kind=self.topic)
+
+        new_node.thumbnail_encoding = '{"base64": "%s"}' % self.thumbnail_data
+
+        assert new_node.get_thumbnail() == self.thumbnail_data
+
+    def test_get_node_thumbnail_file(self):
+        new_node = ContentNode.objects.create(title="Heyo!", parent=self.channel.main_tree, kind=self.topic)
+        thumbnail_file = create_thumbnail_from_base64(testdata.base64encoding())
+        thumbnail_file.contentnode = new_node
+
+        # we need to make sure the file is marked as a thumbnail
+        preset, _created = FormatPreset.objects.get_or_create(id="video_thumbnail")
+        preset.thumbnail = True
+        thumbnail_file.preset = preset
+        thumbnail_file.save()
+
+        assert new_node.get_thumbnail() == generate_storage_url(str(thumbnail_file))
+
+    def test_get_node_details(self):
+        details = self.channel.main_tree.get_details()
+        assert details['resource_count'] > 0
+        assert details['resource_size'] > 0
+        assert details['kind_count'] > 0
+
+
+
 class NodeOperationsTestCase(BaseTestCase):
 
     def setUp(self):

contentcuration/contentcuration/tests/test_files.py

@@ -11,7 +11,9 @@
 from le_utils.constants import format_presets
 from mock import patch
 
+from .testdata import base64encoding
 from .testdata import fileobj_video
+from .testdata import generated_base64encoding
 from .testdata import node
 from contentcuration.management.commands.exportchannel import create_associated_thumbnail
 from contentcuration.models import AssessmentItem
@@ -31,40 +33,6 @@
 pytestmark = pytest.mark.django_db
 
 
-def base64encoding():
-    return "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/" \
-        "9hAAACk0lEQVR4AaWTA7TbbABA8/+zreMdzbYOZtu2bbt4rpPUtvlebbezbdvMvsxmG99740" \
-        "CDF6z4p/G3RYkFLQPGmvj8mx30m7uo1LhNO6ou50r++zrkMoj/cRWUJmIz0gvEDXIVvP/Hbd" \
-        "xRte+chaXe7gDDsP9WwqLJixicgqWwsNrncZFJ2UnmM+Xy1awlqDz/LVsKC6oDtxA0k/B1aD" \
-        "Oi6rMBVVi2ys1Td+qd5NU8ZV0cWEKeWsZ4IKbdn3ikOJTogm9bw1PWw50twAWNFbS9oK1UlX" \
-        "Y337KA6sxwiBb/NIJYM3KrRNOSppD1YNtM9wwHUs+S188M38hXtCKKNSOAM4PmzKCgWQhaNU" \
-        "SiGCIE1DKGYozyJc5EW47ZZ2Ka3U0oNieTbLNjruOHsCO3LvNgq6cZznAHuAICah5DohjDUEG" \
-        "+OciQRsbQlFGKUOvrw9d6uSiiKcu3h9S86F7Me/oMtv/yFVsofaQCYHyhxtcLuFSGNDwatCGI" \
-        "SrZE6EzXIJYkoqILPR0k2oCMo/b1EOpcQqEnjkXPnseOX71uEuqDvQCTAqfjW5fhGkQlWyMQf" \
-        "acZYRHs61jc4HKOJAGXBE+1F1vjdRiwegEstrywB9OYK5zdITZH6xUHTnUADgLcpaBZD1omxCY" \
-        "5m6K7HRaEUDxDZjoyWOs9Xwu/43lbWTUKSfwwzNGfROX2hvg2wGrLjEcGIwTHTHR3sQW0jSEcIN" \
-        "tsnembjYu2z0fKfngHaEXm2jzYmXaUHL7k3H+z6YftOxagZXEXNJ2+eJV3zGF/8RZyWZ6RakH8ad" \
-        "Z9AksmLmz6nO2cy/3vl9+CnJdYZJRmn+x1HsOOh07BkcTF0p/z39hBuoJNuW9U2nF01rngydo/+xr" \
-        "/aXwDY2vpQfdHLrIAAAAASUVORK5CYII="
-
-
-def generated_base64encoding():
-    return "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAA"\
-        "C8klEQVR4nKWTSWhVZxiGv/N//3+Ge+49d8gdkphYOyxMGm+p1QQSm40KIgqKoKUuKu0idFMIWRWKC7"\
-        "G4sqEDxZUEQciwMsaEihsV0ThAojYmahK8NjXJzXCH3DP955zfRUkWIljwW368z7t6H+nA953wPkf/b"\
-        "/DY/q0MACIAUO4bnuTrfwIAwH0X9UTM+OSL7dKb4KFPU9Kh9g8ahBDtAKC8WqO+Ho8ZrucgAIAkhJC6"\
-        "zl047vju54js1MzD8eI6vHtfS0va0I44+bmX3DMvXL45V/wom435vndSQfnB04djF6WfzvXt9aXgBxb"\
-        "RB6iqNpZWV36ZvD+62PH1gSqf0SEvpGY5wp6Lf/TebtjRkonEE53ctie8cuUoCtJNiAMdOgsPVyU3fUm"\
-        "Z/CTOcNf21tbs7D/zjYvLhUaUCP04lc5kdzZ/FmfYSpk8lUpuatNZeJg40EE0IddIHJaE6WC9oj1Kx5Lf"\
-        "ZKJxHhipr1aAGWElJEdQOVifTnupWPJEvaKNB6YjS1zkNaHUEtlDP6ongNhQ8ktmFboiT/9dnTYkLZWK"\
-        "1wLSEHBHqm6qrp1BVyz7RTNObChF9YSQPSII9SQURdOkXNSU14ICA9RIItlCLNtEywaVIKgEvelcvpUB"\
-        "yuVKUKZcVIuCZVGPEEpc8QgLvAkU/7aqhL9Np5PdC6X8i9LL3ChW7OMFRmmFkDFC6eNUNPOrbS19xx3n"\
-        "Fhb5NvCDMaIw9TcU0i6yYBZDhnGl7LHZ/it9eevVUq81lx99MZWbnsnN9/SPDCys+Ww2FDGGyEJlDQVpU5"\
-        "j6OxnMjUwIHvzMLTv0bOT61Z6B7mUAACVeh9FYnbpl81btw6ZmDQCgZ6B76flfN65yy9EE908P5kYmKQDA0"\
-        "OK1Ozu9htH7dEqsjyik6O0RVW/KIFM8yzoMABMAAPdg0m1exD/v4t9iY8oAAPfokw34v4JwjcxkQYIAYq5b9"\
-        "+OJrg1v1uF3yITnGcV5zxcxRYhLZ3rOem9LSe+r82vB1kP1vFwEDQAAAABJRU5ErkJggg=="
-
-
 class FileSaveTestCase(BaseAPITestCase):
 
     def setUp(self):