From 867ea8d469342c95ade88744746a284271af5ee5 Mon Sep 17 00:00:00 2001 From: David Ormsbee Date: Fri, 10 Feb 2023 16:18:42 -0500 Subject: [PATCH] feat: add Component-level versioning MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is a major simplification and refactoring of the data model code in order to get to something that we might be able to build v2 content library tagging on top of (the first tagging implementation will be strictly on Components and not Units). Some high level points: * Creates the openedx_learning.core.components app where Component content data models are implemented. * Adds the PublishLogEntry model to openedx_learning.core.publishing to track when various things are published in a central location. * Renames things to align better with existing Studio conventions, like "Components" instead of "Items". * Removes some aspirational-but-not-really-implemented modules like data.py and api.py for apps, as well as apps that didn't have real implementations yet (staticassets, composition). * Removes LearningPackage-level versioning in favor of versioning at the Component level. This makes per-version storage overhead smaller at the cost of making "point-in-time" snapshots more difficult to construct–you'd have to look at ComponentPublishLogEntry (joined with PublishLogEntry) to look for the most recent component_version before a certain time for each component. Certainly possible, but more expensive. --- .gitignore | 4 +- .importlinter | 5 +- README.rst | 2 +- docs/decisions/0002-content-flexibility.rst | 21 +- .../management/commands/load_components.py | 219 ++++++++++ .../management/commands/load_course_data.py | 236 ---------- .../contrib/staticassets/admin.py | 3 - openedx_learning/contrib/staticassets/apps.py | 6 - .../staticassets/migrations/0001_initial.py | 32 -- .../contrib/staticassets/models.py | 23 - .../contrib/staticassets/tests.py | 3 - .../contrib/staticassets/views.py | 3 - .../components}/__init__.py | 0 openedx_learning/core/components/admin.py | 256 +++++++++++ openedx_learning/core/components/apps.py | 11 + .../components/migrations/0001_initial.py | 336 +++++++++++++++ .../components}/migrations/__init__.py | 0 openedx_learning/core/components/models.py | 403 ++++++++++++++++++ openedx_learning/core/components/readme.rst | 22 + openedx_learning/core/composition/apps.py | 14 - openedx_learning/core/composition/models.py | 93 ---- openedx_learning/core/itemstore/apps.py | 10 - .../core/itemstore/migrations/0001_initial.py | 113 ----- .../core/itemstore/migrations/__init__.py | 0 openedx_learning/core/itemstore/models.py | 233 ---------- openedx_learning/core/itemstore/models_api.py | 21 - openedx_learning/core/publishing/admin.py | 38 +- openedx_learning/core/publishing/api.py | 49 --- openedx_learning/core/publishing/apps.py | 3 +- .../publishing/migrations/0001_initial.py | 88 +++- openedx_learning/core/publishing/models.py | 70 ++- openedx_learning/core/publishing/readme.rst | 4 +- openedx_learning/lib/fields.py | 9 +- .../composition => rest_api}/__init__.py | 0 openedx_learning/rest_api/apps.py | 11 + openedx_learning/rest_api/urls.py | 4 + .../itemstore => rest_api/v1}/__init__.py | 0 openedx_learning/rest_api/v1/components.py | 28 ++ openedx_learning/rest_api/v1/urls.py | 7 + projects/dev.py | 10 +- projects/urls.py | 2 + requirements/base.in | 11 +- requirements/base.txt | 23 +- requirements/ci.txt | 20 +- requirements/dev.txt | 63 ++- requirements/doc.txt | 42 +- requirements/quality.txt | 54 ++- requirements/test.txt | 35 +- test_settings.py | 6 +- tests/test_models.py | 4 +- 50 files changed, 1643 insertions(+), 1007 deletions(-) create mode 100644 olx_importer/management/commands/load_components.py delete mode 100644 olx_importer/management/commands/load_course_data.py delete mode 100644 openedx_learning/contrib/staticassets/admin.py delete mode 100644 openedx_learning/contrib/staticassets/apps.py delete mode 100644 openedx_learning/contrib/staticassets/migrations/0001_initial.py delete mode 100644 openedx_learning/contrib/staticassets/models.py delete mode 100644 openedx_learning/contrib/staticassets/tests.py delete mode 100644 openedx_learning/contrib/staticassets/views.py rename openedx_learning/{contrib/staticassets => core/components}/__init__.py (100%) create mode 100644 openedx_learning/core/components/admin.py create mode 100644 openedx_learning/core/components/apps.py create mode 100644 openedx_learning/core/components/migrations/0001_initial.py rename openedx_learning/{contrib/staticassets => core/components}/migrations/__init__.py (100%) create mode 100644 openedx_learning/core/components/models.py create mode 100644 openedx_learning/core/components/readme.rst delete mode 100644 openedx_learning/core/composition/apps.py delete mode 100644 openedx_learning/core/composition/models.py delete mode 100644 openedx_learning/core/itemstore/apps.py delete mode 100644 openedx_learning/core/itemstore/migrations/0001_initial.py delete mode 100644 openedx_learning/core/itemstore/migrations/__init__.py delete mode 100644 openedx_learning/core/itemstore/models.py delete mode 100644 openedx_learning/core/itemstore/models_api.py delete mode 100644 openedx_learning/core/publishing/api.py rename openedx_learning/{core/composition => rest_api}/__init__.py (100%) create mode 100644 openedx_learning/rest_api/apps.py create mode 100644 openedx_learning/rest_api/urls.py rename openedx_learning/{core/itemstore => rest_api/v1}/__init__.py (100%) create mode 100644 openedx_learning/rest_api/v1/components.py create mode 100644 openedx_learning/rest_api/v1/urls.py diff --git a/.gitignore b/.gitignore index c90e1e6b..f642f3b2 100644 --- a/.gitignore +++ b/.gitignore @@ -62,4 +62,6 @@ requirements/private.txt # database file dev.db -s + +.vscode + diff --git a/.importlinter b/.importlinter index c00ad940..82d88192 100644 --- a/.importlinter +++ b/.importlinter @@ -29,7 +29,7 @@ layers= # This is layering within our Core apps. # # The lowest layer is "publishing", which holds the basic primitives needed to -# create LearningContexts and versioning. +# create LearningPackages and versioning. # # One layer above that is "itemstore" which stores single Items (e.g. Problem, # Video). @@ -40,6 +40,5 @@ layers= name = Core App Dependency Layering type = layers layers= - openedx_learning.core.composition - openedx_learning.core.itemstore + openedx_learning.core.components openedx_learning.core.publishing diff --git a/README.rst b/README.rst index dc8402fe..566537c0 100644 --- a/README.rst +++ b/README.rst @@ -26,7 +26,7 @@ Parts ~~~~~ * ``openedx_learning.lib`` is for shared utilities, and may include things like custom field types, plugin registration code, etc. -* ``openedx_learning.core`` contains our Core Django apps. +* ``openedx_learning.core`` contains our Core Django apps, where foundational data structures and APIs will live. App Dependencies ~~~~~~~~~~~~~~~~ diff --git a/docs/decisions/0002-content-flexibility.rst b/docs/decisions/0002-content-flexibility.rst index 2166349a..12c9a123 100644 --- a/docs/decisions/0002-content-flexibility.rst +++ b/docs/decisions/0002-content-flexibility.rst @@ -18,18 +18,11 @@ Decision The following are foundational, extensible concepts in the Learning Core, which can be combined in different ways: -Item - An Item is a small piece of content, like a video, problem, or bit of HTML text. It has an identity, renderer, and potentially student state. It is not a container for other content, and has no child elements. - - Items are analogous to the "Module" portion of the traditional Open edX course. - -Segment - A Segment is an ordered list of Items that must be presented to the user together. The Items inside a Segment may be of different types, but it does not make sense to show one of these Items in isolation. An example could be one Item that explains a problem scenario, along with a problem Item that asks a question about it–a common scenario in content libraries. By default, each Item is its own Segment. - - Open edX currently models these as nested Verticals (a.k.a. Units), but this often causes problems for code that traverses the content without realizing that such a nesting is possible. +Component + A Component is a small piece of content, like a video, problem, or bit of HTML text. It has an identity, renderer, and potentially student state. It is not a container for other content, and has no child elements. Unit - This is a list of one or more Segments that is displayed to the user on one page. A Unit may be stitched together using content that comes from multiple sources, such as content libraries. Units do not have to be strictly instructional content, as things like upgrade offers and error messages may also be injected. + A Unit is an ordered list of one or more Components that is typically displayed together. A common use case might be to display some introductory Text, a Video, and some followup Problem (all separate Components). An individual Component in a Unit may or may not make sense when taken outside of that Unit–e.g. a Video may be reusable elsewhere, but the Problem referencing the video might not be. Sequence A Sequence is a collection of Units that are presented one after the other, either to assess student understanding or to achieve some learning objective. @@ -50,3 +43,11 @@ Consequences This is aligned with the ADR on the `Role of XBlock `_, which envisions XBlocks as leaf nodes of instructional content like Videos and Problems, and not as container structures like Units or Sequences. To realize the benefits of this system would require significant changes to Studio and especially the LMS. In particular, this would involve gradually removing the XBlock runtime from much of the courseware logic. This would allow for substantial simplifications of the LMS XBlock runtime itself, such as removing field inheritance. + +Changelog +--------- + +2023-02-06: + +* Renamed "Item" to "Component" to be consistent with user-facing Studio terminology. +* Collapsed the role of Segment into Unit simplify the data model. diff --git a/olx_importer/management/commands/load_components.py b/olx_importer/management/commands/load_components.py new file mode 100644 index 00000000..1e06df71 --- /dev/null +++ b/olx_importer/management/commands/load_components.py @@ -0,0 +1,219 @@ +""" +Quick and hacky management command to dump Component data into our model for +experimentation purposes. This lives in its own app because it's not intended to +be a part of this repo in the longer term. Think of this as just an example app +to validate the data model can do what we need it to do. + +This script manipulates the data models directly, instead of using stable API +calls. This is only because those APIs haven't been created yet, and this is +trying to validate basic questions about the data model. This is not how apps +are intended to use openedx-learning in the longer term. + +Open Question: If the data model is extensible, how do we know whether a change +has really happened between what's currently stored/published for a particular +item and the new value we want to set? For Content that's easy, because we have +actual hashes of the data. But it's not clear how that would work for something +like an ComponentVersion. We'd have to have some kind of mechanism where every +pp that wants to attach data gets to answer the question of "has anything +changed?" in order to decide if we really make a new ComponentVersion or not. +""" +from datetime import datetime, timezone +import codecs +import logging +import mimetypes +import pathlib +import re +import xml.etree.ElementTree as ET + +from django.core.management.base import BaseCommand +from django.db import transaction + +from openedx_learning.core.publishing.models import LearningPackage, PublishLogEntry +from openedx_learning.core.components.models import ( + Content, Component, ComponentVersion, ComponentVersionContent, + ComponentPublishLogEntry, PublishedComponent, +) +from openedx_learning.lib.fields import create_hash_digest + +SUPPORTED_TYPES = ['problem', 'video', 'html'] +logger = logging.getLogger(__name__) + + +class Command(BaseCommand): + help = 'Load sample Component data from course export' + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.learning_package = None + self.course_data_path = None + self.init_known_types() + + def init_known_types(self): + """Intialize mimetypes with some custom mappings we want to use.""" + # This is our own hacky video transcripts related format. + mimetypes.add_type("application/vnd.openedx.srt+json", ".sjson") + + # Python's stdlib doesn't include these files that are sometimes used. + mimetypes.add_type("text/markdown", ".md") + mimetypes.add_type("image/svg+xml", ".svg") + + # Historically, JavaScript was "application/javascript", but it's now + # officially "text/javascript" + mimetypes.add_type("text/javascript", ".js") + mimetypes.add_type("text/javascript", ".mjs") + + + def add_arguments(self, parser): + parser.add_argument('course_data_path', type=pathlib.Path) + parser.add_argument('learning_package_identifier', type=str) + + def handle(self, course_data_path, learning_package_identifier, **options): + self.course_data_path = course_data_path + self.learning_package_identifier = learning_package_identifier + self.load_course_data(learning_package_identifier) + + def get_course_title(self): + course_type_dir = self.course_data_path / 'course' + course_xml_file = next(course_type_dir.glob('*.xml')) + course_root = ET.parse(course_xml_file).getroot() + return course_root.attrib.get("display_name", "Unknown Course") + + def load_course_data(self, learning_package_identifier): + print(f"Importing course from: {self.course_data_path}") + now = datetime.now(timezone.utc) + title = self.get_course_title() + + with transaction.atomic(): + learning_package, _created = LearningPackage.objects.get_or_create( + identifier=learning_package_identifier, + defaults={ + 'title': title, + 'created': now, + 'updated': now, + }, + ) + self.learning_package = learning_package + + publish_log_entry = PublishLogEntry.objects.create( + learning_package=learning_package, + message="Initial Import", + published_at=now, + published_by=None, + ) + + for block_type in SUPPORTED_TYPES: + self.import_block_type(block_type, now, publish_log_entry) + + def create_content(self, static_local_path, now, component_version): + identifier = pathlib.Path('static') / static_local_path + real_path = self.course_data_path / identifier + mime_type, _encoding = mimetypes.guess_type(identifier) + if mime_type is None: + logger.error(f" no mimetype found for {real_path}, defaulting to application/binary") + mime_type = "application/binary" + + try: + data_bytes = real_path.read_bytes() + except FileNotFoundError: + logger.warning(f" Static reference not found: {real_path}") + return # Might as well bail if we can't find the file. + + hash_digest = create_hash_digest(data_bytes) + + content, _created = Content.objects.get_or_create( + learning_package=self.learning_package, + mime_type=mime_type, + hash_digest=hash_digest, + defaults = dict( + data=data_bytes, + size=len(data_bytes), + created=now, + ) + ) + ComponentVersionContent.objects.get_or_create( + component_version=component_version, + content=content, + identifier=identifier, + ) + + def import_block_type(self, block_type, now, publish_log_entry): + components_found = 0 + + # Find everything that looks like a reference to a static file appearing + # in attribute quotes, stripping off the querystring at the end. This is + # not fool-proof as it will match static file references that are + # outside of tag declarations as well. + static_files_regex = re.compile(r"""['"]\/static\/(.+?)["'\?]""") + block_data_path = self.course_data_path / block_type + + for xml_file_path in block_data_path.glob('*.xml'): + components_found += 1 + identifier = xml_file_path.stem + + # Find or create the Component itself + component, _created = Component.objects.get_or_create( + learning_package=self.learning_package, + namespace='xblock.v1', + type=block_type, + identifier=identifier, + defaults = { + 'created': now, + } + ) + + # Create the Content entry for the raw data... + data_bytes = xml_file_path.read_bytes() + hash_digest = create_hash_digest(data_bytes) + data_str = codecs.decode(data_bytes, 'utf-8') + content, _created = Content.objects.get_or_create( + learning_package=self.learning_package, + mime_type=f'application/vnd.openedx.xblock.v1.{block_type}+xml', + hash_digest=hash_digest, + defaults = dict( + data=data_bytes, + size=len(data_bytes), + created=now, + ) + ) + # TODO: Get associated file contents, both with the static regex, as + # well as with XBlock-specific code that examines attributes in + # video and HTML tag definitions. + + try: + block_root = ET.fromstring(data_str) + except ET.ParseError as err: + logger.error(f"Parse error for {xml_file_path}: {err}") + continue + + display_name = block_root.attrib.get('display_name', "") + + # Create the ComponentVersion + component_version = ComponentVersion.objects.create( + component=component, + version_num=1, # This only works for initial import + title=display_name, + created=now, + created_by=None, + ) + ComponentVersionContent.objects.create( + component_version=component_version, + content=content, + identifier='source.xml', + ) + static_files_found = static_files_regex.findall(data_str) + for static_local_path in static_files_found: + self.create_content(static_local_path, now, component_version) + + # Mark that Component as Published + component_publish_log_entry = ComponentPublishLogEntry.objects.create( + component=component, + component_version=component_version, + publish_log_entry=publish_log_entry, + ) + PublishedComponent.objects.create( + component=component, + component_version=component_version, + component_publish_log_entry=component_publish_log_entry, + ) + + print(f"{block_type}: {components_found}") diff --git a/olx_importer/management/commands/load_course_data.py b/olx_importer/management/commands/load_course_data.py deleted file mode 100644 index 5e7cf833..00000000 --- a/olx_importer/management/commands/load_course_data.py +++ /dev/null @@ -1,236 +0,0 @@ -""" -Quick and hacky management command to dump course data into our model for -experimentation purposes. This lives in its own app because it's not intended to -be a part of this repo in the longer term. Think of this as just an example app -to validate the data model can do what we need it to do. - -This script manipulates the data models directly, instead of using stable API -calls. This is only because those APIs haven't been created yet, and this is -trying to validate basic questions about the data model. This is not how apps -are intended to use openedx-learning in the longer term. - -Open Question: If the data model is extensible, how do we know whether a change -has really happened between what's currently stored/published for a particular -item and the new value we want to set? For Content that's easy, because we have -actual hashes of the data. But it's not clear how that would work for something -like an ItemVersion. We'd have to have some kind of mechanism where every app -that wants to attach data gets to answer the question of "has anything changed?" -in order to decide if we really make a new ItemVersion or not. -""" -from collections import defaultdict, Counter -from datetime import datetime, timezone -import codecs -import logging -import mimetypes -import pathlib -import xml.etree.ElementTree as ET - -from django.core.management.base import BaseCommand -from django.db import transaction - -from openedx_learning.contrib.staticassets.models import Asset, ItemVersionAsset -from openedx_learning.core.publishing.models import ( - LearningContext, LearningContextVersion -) -from openedx_learning.core.itemstore.models import ( - Content, Item, ItemVersion, LearningContextVersionItemVersion -) -from openedx_learning.lib.fields import create_hash_digest - -SUPPORTED_TYPES = ['lti', 'problem', 'video'] -logger = logging.getLogger(__name__) - - -class Command(BaseCommand): - help = 'Load sample data' - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.learning_context = None - self.init_known_types() - - def init_known_types(self): - """Intialize mimetypes with some custom mappings we want to use.""" - mimetypes.add_type("application/vnd.openedx.srt+json", ".sjson") - mimetypes.add_type("text/markdown", ".md") - - def add_arguments(self, parser): - parser.add_argument('learning_context_identifier', type=str) - parser.add_argument('course_data_path', type=pathlib.Path) - - def handle(self, learning_context_identifier, course_data_path, **options): - self.learning_context_identifier = learning_context_identifier - self.load_course_data(learning_context_identifier, course_data_path) - - def load_course_data(self, learning_context_identifier, course_data_path): - print(f"Importing course from: {course_data_path}") - now = datetime.now(timezone.utc) - - with transaction.atomic(): - learning_context, _created = LearningContext.objects.get_or_create( - identifier=learning_context_identifier, - defaults={'created': now}, - ) - self.learning_context = learning_context - - # For now, create always create a new LearningContextVersion (in the - # future, we need to be careful about detecting changes). - self.new_lcv = LearningContextVersion.objects.create( - learning_context=learning_context, - prev_version=None, - created=now, - ) - - # Future Note: - # Make the static asset loading happen after XBlock loading - # Make the static asset piece grep through created content. - existing_item_raws = Content.objects \ - .filter(learning_context=learning_context) \ - .values_list('id', 'type', 'sub_type', 'hash_digest') - item_raw_id_cache = { - (f"{type}/{sub_type}", hash_digest): item_raw_id - for item_raw_id, type, sub_type, hash_digest in existing_item_raws - } - - static_asset_paths_to_atom_ids = self.import_static_assets( - course_data_path, - item_raw_id_cache, - now, - ) - - for block_type in SUPPORTED_TYPES: - self.import_block_type( - block_type, - course_data_path / block_type, - static_asset_paths_to_atom_ids, - item_raw_id_cache, - now, - ) - - - def import_static_assets(self, course_data_path, item_raw_id_cache, now): - IGNORED_NAMES = [".DS_Store"] - static_assets_path = course_data_path / "static" - file_paths = ( - fp for fp in static_assets_path.glob("**/*") - if fp.is_file() and fp.stem not in IGNORED_NAMES - ) - - num_files = 0 - cum_size = 0 - mime_types_seen = Counter() - paths_to_item_raw_ids = {} - longest_identifier_len = 0 - print("Reading static assets...\n") - - for file_path in file_paths: - identifier = str(file_path.relative_to(course_data_path)) - longest_identifier_len = max(longest_identifier_len, len(str(identifier))) - - data_bytes = file_path.read_bytes() - - num_files += 1 - cum_size += len(data_bytes) - print(f"Static file #{num_files}: ({(cum_size / 1_000_000):.2f} MB)", end="\r") - - data_hash = create_hash_digest(data_bytes) - if file_path.suffix == "": - mime_type = "text/plain" - else: - mime_type, _encoding = mimetypes.guess_type(identifier) - - mime_types_seen[mime_type] += 1 - - if mime_type is None: - print(identifier) - - item_raw_id = item_raw_id_cache.get((mime_type, data_hash)) - if item_raw_id is None: - type, sub_type = mime_type.split('/') - item_raw, _created = Content.objects.get_or_create( - learning_context=self.learning_context, - type=type, - sub_type=sub_type, - hash_digest=data_hash, - defaults={ - 'data': data_bytes, - 'size': len(data_bytes), - 'created': now, - } - ) - item_raw_id = item_raw.id - - paths_to_item_raw_ids[identifier] = item_raw_id - - print(f"{num_files} assets, totaling {(cum_size / 1_000_000):.2f} MB") - print(f"Longest identifier length seen: {longest_identifier_len}") - print("MIME types seen:") - for mime_type_str, freq in sorted(mime_types_seen.items()): - print(f"* {mime_type_str}: {freq}") - - return paths_to_item_raw_ids - - - def import_block_type(self, block_type, content_path, static_asset_paths_to_atom_ids, item_raw_id_cache, now): - items_found = 0 - - # Find everything that looks like a reference to a static file appearing - # in attribute quotes, stripping off the querystring at the end. This is - # not fool-proof as it will match static file references that are - # outside of tag declarations as well. - static_files_regex = r"""['"]\/static\/(.+?)["'\?]""" - - for xml_file_path in content_path.iterdir(): - items_found += 1 - identifier = xml_file_path.stem - - # Find or create the Item itself - item, _created = Item.objects.get_or_create( - learning_context=self.learning_context, - namespace='xblock.v1', - identifier=identifier, - defaults = { - 'created': now, - 'modified': now, - } - ) - - # Create the Content entry for the raw data... - data_bytes = xml_file_path.read_bytes() - hash_digest = create_hash_digest(data_bytes) - data_str = codecs.decode(data_bytes, 'utf-8') - mime_type = f'application/vnd.openedx.xblock.v1.{block_type}+xml' - content, _created = Content.objects.get_or_create( - learning_context=self.learning_context, - mime_type=mime_type, - hash_digest=hash_digest, - defaults = dict( - data=data_bytes, - size=len(data_bytes), - created=now, - ) - ) - - try: - block_root = ET.fromstring(data_str) - except ET.ParseError as err: - logger.error(f"Parse error for {xml_file_path}: {err}") - continue - - display_name = block_root.attrib.get('display_name', "") - - # Create the ItemVersion - item_version = ItemVersion.objects.create( - item=item, - title=display_name, - created=now, - ) - item_version.contents.add(content) - - LearningContextVersionItemVersion.objects.create( - learning_context_version=self.new_lcv, - item_version=item_version, - item=item, - ) - - print(f"{block_type}: {items_found}") diff --git a/openedx_learning/contrib/staticassets/admin.py b/openedx_learning/contrib/staticassets/admin.py deleted file mode 100644 index 8c38f3f3..00000000 --- a/openedx_learning/contrib/staticassets/admin.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.contrib import admin - -# Register your models here. diff --git a/openedx_learning/contrib/staticassets/apps.py b/openedx_learning/contrib/staticassets/apps.py deleted file mode 100644 index 16076f46..00000000 --- a/openedx_learning/contrib/staticassets/apps.py +++ /dev/null @@ -1,6 +0,0 @@ -from django.apps import AppConfig - - -class StaticAssetsConfig(AppConfig): - default_auto_field = 'django.db.models.BigAutoField' - name = 'openedx_learning.contrib.staticassets' diff --git a/openedx_learning/contrib/staticassets/migrations/0001_initial.py b/openedx_learning/contrib/staticassets/migrations/0001_initial.py deleted file mode 100644 index c611f5fe..00000000 --- a/openedx_learning/contrib/staticassets/migrations/0001_initial.py +++ /dev/null @@ -1,32 +0,0 @@ -# Generated by Django 3.2.10 on 2022-08-07 17:00 - -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - initial = True - - dependencies = [ - ('itemstore', '0001_initial'), - ] - - operations = [ - migrations.CreateModel( - name='Asset', - fields=[ - ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ], - ), - migrations.CreateModel( - name='ItemVersionAsset', - fields=[ - ('item_version', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, primary_key=True, serialize=False, to='itemstore.itemversion')), - ('asset', models.ForeignKey(on_delete=django.db.models.deletion.RESTRICT, to='staticassets.asset')), - ], - options={ - 'abstract': False, - }, - ), - ] diff --git a/openedx_learning/contrib/staticassets/models.py b/openedx_learning/contrib/staticassets/models.py deleted file mode 100644 index 613cbcdf..00000000 --- a/openedx_learning/contrib/staticassets/models.py +++ /dev/null @@ -1,23 +0,0 @@ -""" -Is there enough commonality for this to even make sense as one app, or should -images be treated separately from downloads, e.g. ImageItem, DownloadableItem - -Thought: These separate extension models aren't like subclasses, but more like -aspects. So it's not like an ImageItemVersion is a subclass of DownloadableItem, -but that some ItemVersions will have a downloadable file aspect, and some will -have an image aspect, and all those that have the image aspect will also have -the downloadable piece. -""" -from django.db import models - -from openedx_learning.core.itemstore.models_api import ItemVersionDataMixin - - -class Asset(models.Model): - """ - An Asset may be more than just a single file. - """ - pass - -class ItemVersionAsset(ItemVersionDataMixin): - asset = models.ForeignKey(Asset, on_delete=models.RESTRICT, null=False) diff --git a/openedx_learning/contrib/staticassets/tests.py b/openedx_learning/contrib/staticassets/tests.py deleted file mode 100644 index 7ce503c2..00000000 --- a/openedx_learning/contrib/staticassets/tests.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.test import TestCase - -# Create your tests here. diff --git a/openedx_learning/contrib/staticassets/views.py b/openedx_learning/contrib/staticassets/views.py deleted file mode 100644 index 91ea44a2..00000000 --- a/openedx_learning/contrib/staticassets/views.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.shortcuts import render - -# Create your views here. diff --git a/openedx_learning/contrib/staticassets/__init__.py b/openedx_learning/core/components/__init__.py similarity index 100% rename from openedx_learning/contrib/staticassets/__init__.py rename to openedx_learning/core/components/__init__.py diff --git a/openedx_learning/core/components/admin.py b/openedx_learning/core/components/admin.py new file mode 100644 index 00000000..36e4a799 --- /dev/null +++ b/openedx_learning/core/components/admin.py @@ -0,0 +1,256 @@ +import base64 + +from django.contrib import admin +from django.db.models.aggregates import Count, Sum +from django.template.defaultfilters import filesizeformat +from django.urls import reverse +from django.utils.html import format_html + +from .models import ( + Component, + ComponentVersion, + Content, + PublishedComponent, +) + + +class ReadOnlyModelAdmin(admin.ModelAdmin): + def has_add_permission(self, request): + return False + + def has_change_permission(self, request, obj=None): + return False + + def has_delete_permission(self, request, obj=None): + return False + + +class ComponentVersionInline(admin.TabularInline): + model = ComponentVersion + fields = ["version_num", "created", "title", "format_uuid"] + readonly_fields = ["version_num", "created", "title", "format_uuid"] + extra = 0 + + def format_uuid(self, cv_obj): + return format_html( + '{}', + reverse("admin:components_componentversion_change", args=(cv_obj.id,)), + cv_obj.uuid, + ) + + format_uuid.short_description = "UUID" + + +@admin.register(Component) +class ComponentAdmin(ReadOnlyModelAdmin): + list_display = ("identifier", "uuid", "namespace", "type", "created") + readonly_fields = [ + "learning_package", + "uuid", + "namespace", + "type", + "identifier", + "created", + ] + list_filter = ("type", "learning_package") + search_fields = ["uuid", "identifier"] + inlines = [ComponentVersionInline] + + +@admin.register(PublishedComponent) +class PublishedComponentAdmin(ReadOnlyModelAdmin): + model = PublishedComponent + + def get_queryset(self, request): + queryset = super().get_queryset(request) + return ( + queryset.select_related( + "component", + "component__learning_package", + "component_version", + "component_publish_log_entry__publish_log_entry", + ) + .annotate(size=Sum("component_version__contents__size")) + .annotate(content_count=Count("component_version__contents")) + ) + + readonly_fields = ["component", "component_version", "component_publish_log_entry"] + list_display = [ + "identifier", + "version", + "title", + "published_at", + "type", + "content_count", + "size", + "learning_package", + ] + list_filter = ["component__type", "component__learning_package"] + search_fields = [ + "component__uuid", + "component__identifier", + "component_version__uuid", + "component_version__title", + ] + + def learning_package(self, pc): + return pc.component.learning_package.identifier + + def published_at(self, pc): + return pc.component_publish_log_entry.publish_log_entry.published_at + + def identifier(self, pc): + """ + Link to published ComponentVersion with Component identifier as text. + + This is a little weird in that we're showing the Component identifier, + but linking to the published ComponentVersion. But this is what you want + to link to most of the time, as the link to the Component has almost no + information in it (and can be accessed from the ComponentVersion details + page anyhow). + """ + return format_html( + '{}', + reverse("admin:components_componentversion_change", args=(pc.component_version_id,)), + pc.component.identifier, + ) + + def content_count(self, pc): + return pc.content_count + content_count.short_description = "#" + + def size(self, pc): + return filesizeformat(pc.size) + + def namespace(self, pc): + return pc.component.namespace + + def type(self, pc): + return pc.component.type + + def version(self, pc): + return pc.component_version.version_num + + def title(self, pc): + return pc.component_version.title + + +class ContentInline(admin.TabularInline): + model = ComponentVersion.contents.through + fields = ["format_identifier", "format_size", "rendered_data"] + readonly_fields = ["content", "format_identifier", "format_size", "rendered_data"] + extra = 0 + + def rendered_data(self, cv_obj): + return content_preview(cv_obj.content, 100_000) + + def format_size(self, cv_obj): + return filesizeformat(cv_obj.content.size) + format_size.short_description = "Size" + + def format_identifier(self, cv_obj): + return format_html( + '{}', + reverse("admin:components_content_change", args=(cv_obj.content_id,)), + cv_obj.identifier, + ) + + format_identifier.short_description = "Identifier" + + +@admin.register(ComponentVersion) +class ComponentVersionAdmin(ReadOnlyModelAdmin): + readonly_fields = [ + "component", + "uuid", + "title", + "version_num", + "created", + "contents", + ] + fields = [ + "component", + "uuid", + "title", + "version_num", + "created", + ] + inlines = [ContentInline] + + +@admin.register(Content) +class ContentAdmin(ReadOnlyModelAdmin): + list_display = [ + "hash_digest", + "learning_package", + "mime_type", + "format_size", + "created", + ] + fields = [ + "learning_package", + "hash_digest", + "mime_type", + "format_size", + "created", + "rendered_data", + ] + readonly_fields = [ + "learning_package", + "hash_digest", + "mime_type", + "format_size", + "created", + "rendered_data", + ] + list_filter = ("mime_type", "learning_package") + search_fields = ("hash_digest", "size") + + def format_size(self, content_obj): + return filesizeformat(content_obj.size) + format_size.short_description = "Size" + + def rendered_data(self, content_obj): + return content_preview(content_obj, 10_000_000) + + +def is_displayable_text(mime_type): + # Our usual text files, includiing things like text/markdown, text/html + media_type, media_subtype = mime_type.split('/') + + if media_type == "text": + return True + + # Our OLX goes here, but so do some other things like + if media_subtype.endswith("+xml"): + return True + + # Other application/* types that we know we can display. + if media_subtype in ["json", "x-subrip"]: + return True + + # Other formats that are really specific types of JSON + if media_subtype.endswith("+json"): + return True + + return False + + +def content_preview(content_obj, size_limit): + if content_obj.size > size_limit: + return f"Too large to preview." + + # image before text check, since SVGs can be either, but we probably want to + # see the image version in the admin. + if content_obj.mime_type.startswith("image/"): + b64_str = base64.b64encode(content_obj.data).decode("ascii") + encoded_img_src = f"data:{content_obj.mime_type};base64,{b64_str}" + return format_html('', encoded_img_src) + + if is_displayable_text(content_obj.mime_type): + return format_html( + '
\n{}\n
', + content_obj.data.decode("utf-8"), + ) + + return format_html("This content type cannot be displayed.") diff --git a/openedx_learning/core/components/apps.py b/openedx_learning/core/components/apps.py new file mode 100644 index 00000000..0354fcb7 --- /dev/null +++ b/openedx_learning/core/components/apps.py @@ -0,0 +1,11 @@ +from django.apps import AppConfig + + +class ComponentsConfig(AppConfig): + """ + Configuration for the Components Django application. + """ + + name = "openedx_learning.core.components" + verbose_name = "Learning Core: Components" + default_auto_field = "django.db.models.BigAutoField" diff --git a/openedx_learning/core/components/migrations/0001_initial.py b/openedx_learning/core/components/migrations/0001_initial.py new file mode 100644 index 00000000..273a54ff --- /dev/null +++ b/openedx_learning/core/components/migrations/0001_initial.py @@ -0,0 +1,336 @@ +# Generated by Django 4.1 on 2023-02-10 18:58 + +from django.conf import settings +import django.core.validators +from django.db import migrations, models +import django.db.models.deletion +import uuid + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ("publishing", "0001_initial"), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name="Component", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "uuid", + models.UUIDField( + default=uuid.uuid4, + editable=False, + unique=True, + verbose_name="UUID", + ), + ), + ("namespace", models.CharField(max_length=100)), + ("type", models.CharField(blank=True, max_length=100)), + ("identifier", models.CharField(max_length=255)), + ("created", models.DateTimeField()), + ( + "learning_package", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="publishing.learningpackage", + ), + ), + ], + options={ + "verbose_name": "Component", + "verbose_name_plural": "Components", + }, + ), + migrations.CreateModel( + name="ComponentVersion", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "uuid", + models.UUIDField( + default=uuid.uuid4, + editable=False, + unique=True, + verbose_name="UUID", + ), + ), + ("title", models.CharField(blank=True, default="", max_length=1000)), + ( + "version_num", + models.PositiveBigIntegerField( + validators=[django.core.validators.MinValueValidator(1)] + ), + ), + ("created", models.DateTimeField()), + ( + "component", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="components.component", + ), + ), + ], + options={ + "verbose_name": "Component Version", + "verbose_name_plural": "Component Versions", + }, + ), + migrations.CreateModel( + name="Content", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("hash_digest", models.CharField(editable=False, max_length=40)), + ("mime_type", models.CharField(max_length=255)), + ( + "size", + models.PositiveBigIntegerField( + validators=[django.core.validators.MaxValueValidator(10000000)] + ), + ), + ("created", models.DateTimeField()), + ("data", models.BinaryField(max_length=10000000)), + ( + "learning_package", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="publishing.learningpackage", + ), + ), + ], + ), + migrations.CreateModel( + name="ComponentVersionContent", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("identifier", models.CharField(max_length=255)), + ( + "component_version", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="components.componentversion", + ), + ), + ( + "content", + models.ForeignKey( + on_delete=django.db.models.deletion.RESTRICT, + to="components.content", + ), + ), + ], + ), + migrations.AddField( + model_name="componentversion", + name="contents", + field=models.ManyToManyField( + related_name="component_versions", + through="components.ComponentVersionContent", + to="components.content", + ), + ), + migrations.AddField( + model_name="componentversion", + name="created_by", + field=models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.SET_NULL, + to=settings.AUTH_USER_MODEL, + ), + ), + migrations.CreateModel( + name="ComponentPublishLogEntry", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "component", + models.ForeignKey( + on_delete=django.db.models.deletion.RESTRICT, + to="components.component", + ), + ), + ( + "component_version", + models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.RESTRICT, + to="components.componentversion", + ), + ), + ( + "publish_log_entry", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="publishing.publishlogentry", + ), + ), + ], + ), + migrations.CreateModel( + name="PublishedComponent", + fields=[ + ( + "component", + models.OneToOneField( + on_delete=django.db.models.deletion.RESTRICT, + primary_key=True, + serialize=False, + to="components.component", + ), + ), + ( + "component_publish_log_entry", + models.ForeignKey( + on_delete=django.db.models.deletion.RESTRICT, + to="components.componentpublishlogentry", + ), + ), + ( + "component_version", + models.OneToOneField( + null=True, + on_delete=django.db.models.deletion.RESTRICT, + to="components.componentversion", + ), + ), + ], + options={ + "verbose_name": "Published Component", + "verbose_name_plural": "Published Components", + }, + ), + migrations.AddIndex( + model_name="content", + index=models.Index( + fields=["learning_package", "mime_type"], + name="content_idx_lp_mime_type", + ), + ), + migrations.AddIndex( + model_name="content", + index=models.Index( + fields=["learning_package", "-size"], name="content_idx_lp_rsize" + ), + ), + migrations.AddIndex( + model_name="content", + index=models.Index( + fields=["learning_package", "-created"], name="content_idx_lp_rcreated" + ), + ), + migrations.AddConstraint( + model_name="content", + constraint=models.UniqueConstraint( + fields=("learning_package", "mime_type", "hash_digest"), + name="content_uniq_lc_mime_type_hash_digest", + ), + ), + migrations.AddIndex( + model_name="componentversioncontent", + index=models.Index( + fields=["content", "component_version"], + name="componentversioncontent_c_cv", + ), + ), + migrations.AddIndex( + model_name="componentversioncontent", + index=models.Index( + fields=["component_version", "content"], + name="componentversioncontent_cv_d", + ), + ), + migrations.AddConstraint( + model_name="componentversioncontent", + constraint=models.UniqueConstraint( + fields=("component_version", "identifier"), + name="componentversioncontent_uniq_cv_id", + ), + ), + migrations.AddIndex( + model_name="componentversion", + index=models.Index( + fields=["component", "-created"], name="cv_idx_component_rcreated" + ), + ), + migrations.AddIndex( + model_name="componentversion", + index=models.Index(fields=["title"], name="cv_idx_title"), + ), + migrations.AddConstraint( + model_name="componentversion", + constraint=models.UniqueConstraint( + fields=("component", "version_num"), + name="cv_uniq_component_version_num", + ), + ), + migrations.AddIndex( + model_name="component", + index=models.Index( + fields=["learning_package", "identifier"], + name="component_idx_lp_identifier", + ), + ), + migrations.AddIndex( + model_name="component", + index=models.Index(fields=["identifier"], name="component_idx_identifier"), + ), + migrations.AddIndex( + model_name="component", + index=models.Index( + fields=["learning_package", "-created"], + name="component_idx_lp_rcreated", + ), + ), + migrations.AddConstraint( + model_name="component", + constraint=models.UniqueConstraint( + fields=("learning_package", "namespace", "type", "identifier"), + name="component_uniq_lc_ns_type_identifier", + ), + ), + ] diff --git a/openedx_learning/contrib/staticassets/migrations/__init__.py b/openedx_learning/core/components/migrations/__init__.py similarity index 100% rename from openedx_learning/contrib/staticassets/migrations/__init__.py rename to openedx_learning/core/components/migrations/__init__.py diff --git a/openedx_learning/core/components/models.py b/openedx_learning/core/components/models.py new file mode 100644 index 00000000..fcd284d3 --- /dev/null +++ b/openedx_learning/core/components/models.py @@ -0,0 +1,403 @@ +""" +The model hierarchy is Component -> ComponentVersion -> Content. + +A Component is an entity like a Problem or Video. It has enough information to +identify the Component and determine what the handler should be (e.g. XBlock +Problem), but little beyond that. + +Components have one or more ComponentVersions, which represent saved versions of +that Component. At any time, there is at most one published ComponentVersion for +a Component in a LearningPackage (there can be zero if it's unpublished). The +publish status is tracked in PublishedComponent, with historical publish data in +ComponentPublishLogEntry. + +Content is a simple model holding unversioned, raw data, along with some simple +metadata like size and MIME type. + +Multiple pieces of Content may be associated with a ComponentVersion, through +the ComponentVersionContent model. ComponentVersionContent allows to specify a +Component-local identifier. We're using this like a file path by convention, but +it's possible we might want to have special identifiers later. +""" +from django.db import models +from django.conf import settings +from django.core.validators import MinValueValidator, MaxValueValidator + +from openedx_learning.lib.fields import ( + hash_field, + identifier_field, + immutable_uuid_field, + manual_date_time_field, +) +from ..publishing.models import LearningPackage, PublishLogEntry + + +class Component(models.Model): + """ + This represents any content that has ever existed in a LearningPackage. + + A Component will have many ComponentVersions over time, and most metadata is + associated with the ComponentVersion model. Make a foreign key to this model + when you need a stable reference that will exist for as long as the + LearningPackage itself exists. It is possible for an Component to have no + published ComponentVersion, either because it was never published or because + it's been "deleted" (made unavailable). + + A Component belongs to one and only one LearningPackage. + + The UUID should be treated as immutable. The identifier field *is* mutable, + but changing it will affect all ComponentVersions. If you are referencing + this model from within the same process, use a foreign key to the id. If you + are referencing this Component from an external system, use the UUID. Do NOT + use the identifier if you can help it, since this can be changed. + + Note: When we actually implement the ability to change identifiers, we + should make a history table and a modified attribute on this model. + """ + + uuid = immutable_uuid_field() + learning_package = models.ForeignKey(LearningPackage, on_delete=models.CASCADE) + + # namespace and type work together to help figure out what Component needs + # to handle this data. A namespace is *required*. The namespace for XBlocks + # is "xblock.v1" (to match the setup.py entrypoint naming scheme). + namespace = models.CharField(max_length=100, null=False, blank=False) + + # type is a way to help sub-divide namespace if that's convenient. This + # field cannot be null, but it can be blank if it's not necessary. For an + # XBlock, type corresponds to tag, e.g. "video". It's also the block_type in + # the UsageKey. + type = models.CharField(max_length=100, null=False, blank=True) + + # identifier is local to a learning_package + namespace + type. For XBlocks, + # this is the block_id part of the UsageKey, which usually shows up in the + # OLX as the url_name attribute. + identifier = identifier_field() + + created = manual_date_time_field() + + class Meta: + constraints = [ + # The combination of (namespace, type, identifier) is unique within + # a given LearningPackage. Note that this means it is possible to + # have two Components that have the exact same identifier. An XBlock + # would be modeled as namespace="xblock.v1" with the type as the + # block_type, so the identifier would only be the block_id (the + # very last part of the UsageKey). + models.UniqueConstraint( + fields=[ + "learning_package", + "namespace", + "type", + "identifier", + ], + name="component_uniq_lc_ns_type_identifier", + ) + ] + indexes = [ + # LearningPackage Identifier Index: + # * Search by identifier (without having to specify namespace and + # type). This kind of freeform search will likely be common. + models.Index( + fields=["learning_package", "identifier"], + name="component_idx_lp_identifier", + ), + + # Global Identifier Index: + # * Search by identifier across all Components on the site. This + # would be a support-oriented tool from Django Admin. + models.Index( + fields=["identifier"], + name="component_idx_identifier", + ), + + # LearningPackage (reverse) Created Index: + # * Search for most recently *created* Components for a given + # LearningPackage, since they're the most likely to be actively + # worked on. + models.Index( + fields=["learning_package", "-created"], + name="component_idx_lp_rcreated", + ), + ] + + # These are for the Django Admin UI. + verbose_name = "Component" + verbose_name_plural = "Components" + + def __str__(self): + return f"{self.identifier}" + + +class ComponentVersion(models.Model): + """ + A particular version of a Component. + + This holds the title (because that's versioned information) and the contents + via a M:M relationship with Content via ComponentVersionContent. + + * Each ComponentVersion belongs to one and only one Component. + * ComponentVersions have a version_num that should increment by one with + each new version. + """ + + uuid = immutable_uuid_field() + component = models.ForeignKey(Component, on_delete=models.CASCADE) + + # Blank titles are allowed because some Components are built to be used from + # a particular Unit, and the title would be redundant in that context (e.g. + # a "Welcome" video in a "Welcome" Unit). + title = models.CharField(max_length=1000, default="", null=False, blank=True) + + # The version_num starts at 1 and increments by 1 with each new version for + # a given Component. Doing it this way makes it more convenient for users to + # refer to than a hash or UUID value. + version_num = models.PositiveBigIntegerField( + null=False, + validators=[MinValueValidator(1)], + ) + + # All ComponentVersions created as part of the same publish should have the + # exact same created datetime (not off by a handful of microseconds). + created = manual_date_time_field() + + # User who created the ContentVersion. This can be null if the user is later + # removed. Open edX in general doesn't let you remove users, but we should + # try to model it so that this is possible eventually. + created_by = models.ForeignKey( + settings.AUTH_USER_MODEL, + on_delete=models.SET_NULL, + null=True, + ) + + # The contents hold the actual interesting data associated with this + # ComponentVersion. + contents = models.ManyToManyField( + "Content", + through="ComponentVersionContent", + related_name="component_versions", + ) + + def __str__(self): + return f"v{self.version_num}: {self.title}" + + class Meta: + constraints = [ + # Prevent the situation where we have multiple ComponentVersions + # claiming to be the same version_num for a given Component. This + # can happen if there's a race condition between concurrent editors + # in different browsers, working on the same Component. With this + # constraint, one of those processes will raise an IntegrityError. + models.UniqueConstraint( + fields=[ + "component", + "version_num", + ], + name="cv_uniq_component_version_num", + ) + ] + indexes = [ + # LearningPackage (reverse) Created Index: + # * Make it cheap to find the most recently created + # ComponentVersions for a given LearningPackage. This represents + # the most recently saved work for a LearningPackage and would + # be the most likely areas to get worked on next. + models.Index( + fields=["component", "-created"], + name="cv_idx_component_rcreated", + ), + + # Title Index: + # * Search by title. + models.Index( + fields=["title",], + name="cv_idx_title", + ), + ] + + # These are for the Django Admin UI. + verbose_name = "Component Version" + verbose_name_plural = "Component Versions" + + +class ComponentPublishLogEntry(models.Model): + """ + This is a historical record of Component publishing. + + When a ComponentVersion is initially created, it's considered a draft. The + act of publishing means we're marking a ContentVersion as the official, + ready-for-use version of this Component. + """ + + publish_log_entry = models.ForeignKey(PublishLogEntry, on_delete=models.CASCADE) + component = models.ForeignKey(Component, on_delete=models.RESTRICT) + component_version = models.ForeignKey( + ComponentVersion, on_delete=models.RESTRICT, null=True + ) + + +class PublishedComponent(models.Model): + """ + For any given Component, what is the currently published ComponentVersion. + + It may be possible for a Component to exist only as a Draft (and thus not + show up in this table). There is only ever one published ComponentVersion + per Component at any given time. + + TODO: Do we need to create a (redundant) title field in this model so that + we can more efficiently search across titles within a LearningPackage? + Probably not an immediate concern because the number of rows currently + shouldn't be > 10,000 in the more extreme cases. + """ + + component = models.OneToOneField( + Component, on_delete=models.RESTRICT, primary_key=True + ) + component_version = models.OneToOneField( + ComponentVersion, + on_delete=models.RESTRICT, + null=True, + ) + component_publish_log_entry = models.ForeignKey( + ComponentPublishLogEntry, + on_delete=models.RESTRICT, + ) + + class Meta: + verbose_name = "Published Component" + verbose_name_plural = "Published Components" + + +class Content(models.Model): + """ + This is the most basic piece of raw content data, with no version metadata. + + Content stores data in an immutable Binary BLOB `data` field. This data is + not auto-normalized in any way, meaning that pieces of content that are + semantically equivalent (e.g. differently spaced/sorted JSON) will result in + new entries. This model is intentionally ignorant of what these things mean, + because it expects supplemental data models to build on top of it. + + Two Content instances _can_ have the same hash_digest if they are of + different MIME types. For instance, an empty text file and an empty SRT file + will both hash the same way, but be considered different entities. + + The other fields on Content are for data that is intrinsic to the file data + itself (e.g. the size). Any smart parsing of the contents into more + structured metadata should happen in other models that hang off of Content. + + Content models are not versioned in any way. The concept of versioning only + exists at a higher level. + + Since this model uses a BinaryField to hold its data, we have to be careful + about scalability issues. For instance, video files should not be stored + here directly. There is a 10 MB limit set for the moment, to accomodate + things like PDF files and images, but the itention is for the vast majority + of rows to be much smaller than that. + """ + + # Cap item size at 10 MB for now. + MAX_SIZE = 10_000_000 + + learning_package = models.ForeignKey(LearningPackage, on_delete=models.CASCADE) + hash_digest = hash_field() + + # MIME type, such as "text/html", "image/png", etc. Per RFC 4288, MIME type + # and sub-type may each be 127 chars, making a max of 255 (including the "/" + # in between). + # + # DO NOT STORE parameters here, e.g. "charset=". We can make a new field if + # that becomes necessary. + mime_type = models.CharField(max_length=255, blank=False, null=False) + + size = models.PositiveBigIntegerField( + validators=[MaxValueValidator(MAX_SIZE)], + ) + + # This should be manually set so that multiple Content rows being set in the + # same transaction are created with the same timestamp. The timestamp should + # be UTC. + created = manual_date_time_field() + + data = models.BinaryField(null=False, max_length=MAX_SIZE) + + class Meta: + constraints = [ + # Make sure we don't store duplicates of this raw data within the + # same LearningPackage, unless they're of different MIME types. + models.UniqueConstraint( + fields=[ + "learning_package", + "mime_type", + "hash_digest", + ], + name="content_uniq_lc_mime_type_hash_digest", + ), + ] + indexes = [ + # LearningPackage MIME type Index: + # * Break down Content counts by type/subtype within a + # LearningPackage. + # * Find all the Content in a LearningPackage that matches a + # certain MIME type (e.g. "image/png", "application/pdf". + models.Index( + fields=["learning_package", "mime_type"], + name="content_idx_lp_mime_type", + ), + # LearningPackage (reverse) Size Index: + # * Find largest Content in a LearningPackage. + # * Find the sum of Content size for a given LearningPackage. + models.Index( + fields=["learning_package", "-size"], + name="content_idx_lp_rsize", + ), + # LearningPackage (reverse) Created Index: + # * Find most recently added Content. + models.Index( + fields=["learning_package", "-created"], + name="content_idx_lp_rcreated", + ) + ] + + +class ComponentVersionContent(models.Model): + """ + Determines the Content for a given ComponentVersion. + + An ComponentVersion may be associated with multiple pieces of binary data. + For instance, a Video ComponentVersion might be associated with multiple + transcripts in different languages. + + When Content is associated with an ComponentVersion, it has some local + identifier that is unique within the the context of that ComponentVersion. + This allows the ComponentVersion to do things like store an image file and + reference it by a "path" identifier. + + Content is immutable and sharable across multiple ComponentVersions and even + across LearningPackages. + """ + + component_version = models.ForeignKey(ComponentVersion, on_delete=models.CASCADE) + content = models.ForeignKey(Content, on_delete=models.RESTRICT) + identifier = identifier_field() + + class Meta: + constraints = [ + # Uniqueness is only by ComponentVersion and identifier. If for some + # reason a ComponentVersion wants to associate the same piece of + # content with two different identifiers, that is permitted. + models.UniqueConstraint( + fields=["component_version", "identifier"], + name="componentversioncontent_uniq_cv_id", + ), + ] + indexes = [ + models.Index( + fields=["content", "component_version"], + name="componentversioncontent_c_cv", + ), + models.Index( + fields=["component_version", "content"], + name="componentversioncontent_cv_d", + ), + ] diff --git a/openedx_learning/core/components/readme.rst b/openedx_learning/core/components/readme.rst new file mode 100644 index 00000000..e2752f63 --- /dev/null +++ b/openedx_learning/core/components/readme.rst @@ -0,0 +1,22 @@ +Components App +============== + +The ``components`` app holds the versioned data models for the lowest-level pieces of content that can be stored in Open edX: Components (e.g. XBlocks), as well as the individual pieces of raw data content that they reference. + +Motivation +---------- + +We want a small, extensible model for modeling the smallest pieces of content (e.g. individual blocks of XBlock content), that we will build more complex data on top of, like tagging. + +Intended Use Cases +------------------ + + + +Architecture Guidelines +----------------------- + +* We're keeping nearly unlimited history, so per-version metadata (i.e. the space/time cost of making a new version) must be kept low. +* Do not assume that all Components will be XBlocks. +* Encourage other apps to make models that join to (and add their own metadata to) Component, ComponentVersion, Content, etc. But it should be done in such a way that this app is not aware of them. +* Always preserve the most raw version of the data possible, e.g. OLX, even if XBlocks then extend that with more sophisticated data models. At some point those XBlocks will get deprecated/removed, and we will still want to be able to export the raw data. diff --git a/openedx_learning/core/composition/apps.py b/openedx_learning/core/composition/apps.py deleted file mode 100644 index 3af7a4a4..00000000 --- a/openedx_learning/core/composition/apps.py +++ /dev/null @@ -1,14 +0,0 @@ -""" -Composition App Configuration -""" - -from django.apps import AppConfig - - -class CompositionConfig(AppConfig): - """ - Configuration for the publishing Django application. - """ - name = "openedx_learning.core.composition" - verbose_name = "Learning Core: Composition" - default_auto_field = 'django.db.models.BigAutoField' diff --git a/openedx_learning/core/composition/models.py b/openedx_learning/core/composition/models.py deleted file mode 100644 index 64368423..00000000 --- a/openedx_learning/core/composition/models.py +++ /dev/null @@ -1,93 +0,0 @@ -""" -THIS IS NON-FUNCTIONING scratch code at the moment. - -There are broadly three types of UserPartitions: - -1. Partitions that are global to a LearningContextType. -2. Partitions that are specific to a LearningContextVersion. - -""" -from django.db import models - -# from ..publish.models import LearningObjectVersion - -from openedx_learning.lib.fields import hash_field, identifier_field, immutable_uuid_field - -""" -class LearningContextTypePartitionGroup(models.Model): - learning_context_type = models.ForeignKey(LearningContextType, on_delete=models.CASCADE) - partition_id = models.BigIntegerField(null=False) - group_id = models.BigIntegerField(null=False) - - -class LearningContextVersionPartitionGroup(models.Model): - learning_context_version = models.ForeignKey(LearningContextVersion, on_delete=models.CASCADE) - partition_id = models.BigIntegerField(null=False) - group_id = models.BigIntegerField(null=False) -""" - - -#class Block(models.Model): -# """ -# A Block is the simplest possible piece of content. It has no children.# -# How do we decompose this? Separate tables for different aspects, like grades? -# """ -# -# learning_object_version = models.ForeignKey(LearningObjectVersion, on_delete=models.CASCADE) - - -class Unit(models.Model): - identifier = identifier_field() - - -class UnitVersion(models.Model): - uuid = immutable_uuid_field() - identifier = identifier_field() - hash = hash_field() - - -class UnitContentBlock(models.Model): - - - pass - - - -class Partition(models.Model): - """ - Each row represents a Partition for dividing content into PartitionGroups. - - UserPartitions is a pluggable interface. Some IDs are static (with values - less than 100). Others are dynamic, picking a range between 100 and 2^31-1. - That means that technically, we could use IntegerField instead of - BigIntegerField, but a) that limit isn't actually enforced as far as I can - tell; and b) it's not _that_ much extra storage, so I'm using BigInteger - instead (2^63-1). - - It's a pluggable interface (entry points: openedx.user_partition_scheme, - openedx.dynamic_partition_generator), so there's no "UserPartition" model. - We need to actually link this against the values passed back from the - partitions service in order to map them to names and descriptions. - Any CourseSection or CourseSectionSequence may be associated with any number - of UserPartitionGroups. An individual _user_ may only be in one Group for - any given User Partition, but a piece of _content_ can be associated with - multiple groups. So for instance, for the Enrollment Track user partition, - a piece of content may be associated with both "Verified" and "Masters" - tracks, while a user may only be in one or the other. - """ - partition_num = models.BigIntegerField(null=False) - - -class PartitionGroup(models.Model): - """ - """ - partition = models.ForeignKey(Partition, on_delete=models.CASCADE) - group_num = models.BigIntegerField(null=False) - - class Meta: - constraints = [ - models.UniqueConstraint( - name='unique_ol_publishing_partition_partition_group_num', - fields=['partition_id', 'group_num'], - ), - ] diff --git a/openedx_learning/core/itemstore/apps.py b/openedx_learning/core/itemstore/apps.py deleted file mode 100644 index e003fff5..00000000 --- a/openedx_learning/core/itemstore/apps.py +++ /dev/null @@ -1,10 +0,0 @@ -from django.apps import AppConfig - - -class ItemStoreConfig(AppConfig): - """ - Configuration for the publishing Django application. - """ - name = "openedx_learning.core.itemstore" - verbose_name = "Learning Core: Item Store" - default_auto_field = 'django.db.models.BigAutoField' diff --git a/openedx_learning/core/itemstore/migrations/0001_initial.py b/openedx_learning/core/itemstore/migrations/0001_initial.py deleted file mode 100644 index 51b397ad..00000000 --- a/openedx_learning/core/itemstore/migrations/0001_initial.py +++ /dev/null @@ -1,113 +0,0 @@ -# Generated by Django 3.2.10 on 2022-08-07 17:00 - -import django.core.validators -from django.db import migrations, models -import django.db.models.deletion -import uuid - - -class Migration(migrations.Migration): - - initial = True - - dependencies = [ - ('publishing', '0001_initial'), - ] - - operations = [ - migrations.CreateModel( - name='Content', - fields=[ - ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('hash_digest', models.CharField(editable=False, max_length=40)), - ('type', models.CharField(max_length=127)), - ('sub_type', models.CharField(max_length=127)), - ('size', models.PositiveBigIntegerField(validators=[django.core.validators.MaxValueValidator(10000000)])), - ('created', models.DateTimeField()), - ('data', models.BinaryField(max_length=10000000)), - ('learning_context', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='publishing.learningcontext')), - ], - ), - migrations.CreateModel( - name='Item', - fields=[ - ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('uuid', models.UUIDField(default=uuid.uuid4, editable=False, unique=True)), - ('namespace', models.CharField(max_length=100)), - ('identifier', models.CharField(max_length=255)), - ('created', models.DateTimeField()), - ('modified', models.DateTimeField()), - ('learning_context', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='publishing.learningcontext')), - ], - ), - migrations.CreateModel( - name='ItemVersion', - fields=[ - ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('uuid', models.UUIDField(default=uuid.uuid4, editable=False, unique=True)), - ('title', models.CharField(blank=True, max_length=1000, null=True)), - ('created', models.DateTimeField()), - ], - ), - migrations.CreateModel( - name='LearningContextVersionItemVersion', - fields=[ - ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('item', models.ForeignKey(on_delete=django.db.models.deletion.RESTRICT, to='itemstore.item')), - ('item_version', models.ForeignKey(on_delete=django.db.models.deletion.RESTRICT, to='itemstore.itemversion')), - ('learning_context_version', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='publishing.learningcontextversion')), - ], - ), - migrations.CreateModel( - name='ItemVersionContent', - fields=[ - ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('identifier', models.CharField(max_length=255)), - ('content', models.ForeignKey(on_delete=django.db.models.deletion.RESTRICT, to='itemstore.content')), - ('item_version', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='itemstore.itemversion')), - ], - ), - migrations.AddField( - model_name='itemversion', - name='contents', - field=models.ManyToManyField(related_name='item_versions', through='itemstore.ItemVersionContent', to='itemstore.Content'), - ), - migrations.AddField( - model_name='itemversion', - name='item', - field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='itemstore.item'), - ), - migrations.AddField( - model_name='itemversion', - name='learning_context_versions', - field=models.ManyToManyField(related_name='item_versions', through='itemstore.LearningContextVersionItemVersion', to='publishing.LearningContextVersion'), - ), - migrations.AddConstraint( - model_name='learningcontextversionitemversion', - constraint=models.UniqueConstraint(fields=('learning_context_version_id', 'item_version_id'), name='lcviv_uniq_lcv_iv'), - ), - migrations.AddConstraint( - model_name='learningcontextversionitemversion', - constraint=models.UniqueConstraint(fields=('learning_context_version', 'item'), name='lcviv_uniq_lcv_item'), - ), - migrations.AddIndex( - model_name='itemversioncontent', - index=models.Index(fields=['content', 'item_version'], name='itemversioncontent_c_iv'), - ), - migrations.AddIndex( - model_name='itemversioncontent', - index=models.Index(fields=['item_version', 'content'], name='itemversioncontent_iv_d'), - ), - migrations.AddConstraint( - model_name='itemversioncontent', - constraint=models.UniqueConstraint(fields=('item_version', 'identifier'), name='itemversioncontent_uniq_iv_id'), - ), - migrations.AddConstraint( - model_name='item', - constraint=models.UniqueConstraint(fields=('learning_context', 'namespace', 'identifier'), name='item_uniq_lc_ns_identifier'), - ), - migrations.AddConstraint( - model_name='content', - constraint=models.UniqueConstraint(fields=('learning_context', 'type', 'sub_type', 'hash_digest'), name='content_uniq_lc_hd'), - ), - ] diff --git a/openedx_learning/core/itemstore/migrations/__init__.py b/openedx_learning/core/itemstore/migrations/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/openedx_learning/core/itemstore/models.py b/openedx_learning/core/itemstore/models.py deleted file mode 100644 index 17066309..00000000 --- a/openedx_learning/core/itemstore/models.py +++ /dev/null @@ -1,233 +0,0 @@ -""" -The item model hiearachy is: Item -> ItemVersion -> Content - -Item is the versionless thing that is guaranteed to exist for the lifetime of -the LearningContext. An ItemVersion is a different version of that item for a -given LearningContext, and may include policy changes (like grading). Content -represents the raw byte data. -""" -from django.db import models -from django.core.validators import MaxValueValidator - -from openedx_learning.lib.fields import ( - hash_field, - identifier_field, - immutable_uuid_field, - manual_date_time_field, -) -from ..publishing.models import LearningContext, LearningContextVersion - - -class Item(models.Model): - """ - This represents any content that has ever existed in a LearningContext. - - An Item will have many ItemVersions over time, and most metadata is - associated with the ItemVersion model. Make a foreign key to this model when - you need a stable reference that will exist for as long as the - LearningContext itself exists. It is possible for an Item to have no active - ItemVersion in the current LearningContextVersion (i.e. this content was at - some point removed from the "published" version). - - An Item belongs to one and only one LearningContext. - - The UUID should be treated as immutable. The identifier field *is* mutable, - but changing it will affect all ItemVersions. - """ - uuid = immutable_uuid_field() - learning_context = models.ForeignKey(LearningContext, on_delete=models.CASCADE) - - # namespace/identifier work together to give a more humanly readable, local - # identifier for an Item. - namespace = models.CharField(max_length=100, null=False, blank=False) - identifier = identifier_field() - - created = manual_date_time_field() - modified = manual_date_time_field() - - class Meta: - constraints = [ - models.UniqueConstraint( - fields=["learning_context", "namespace", "identifier"], - name="item_uniq_lc_ns_identifier", - ) - ] - - def __str__(self): - return f"{self.identifier}" - - -class ItemVersion(models.Model): - """ - A particular version of an Item. - - A new ItemVersion should be created anytime there is either a change to the - content or a change to the policy around a piece of content (e.g. schedule - change). - - Each ItemVersion belongs to one and only one Item. - - TODO: created_by field? - """ - uuid = immutable_uuid_field() - item = models.ForeignKey(Item, on_delete=models.CASCADE) - - title = models.CharField(max_length=1000, blank=True, null=True) - created = manual_date_time_field() - - learning_context_versions = models.ManyToManyField( - LearningContextVersion, - through='LearningContextVersionItemVersion', - related_name='item_versions', - ) - contents = models.ManyToManyField( - 'Content', - through='ItemVersionContent', - related_name='item_versions', - ) - - def __str__(self): - return f"{self.uuid}: {self.title}" - - -class LearningContextVersionItemVersion(models.Model): - """ - Mapping of all ItemVersion in a given LearningContextVersion. - """ - learning_context_version = models.ForeignKey(LearningContextVersion, on_delete=models.CASCADE) - item_version = models.ForeignKey(ItemVersion, on_delete=models.RESTRICT) - - # item should always be derivable from item_version, but it exists in this - # model directly because MySQL doesn't support constraint conditions (see - # comments in the constraints section below for details). - item = models.ForeignKey(Item, on_delete=models.RESTRICT) - - class Meta: - constraints = [ - # The same ItemVersion should only show up once for a given - # LearningContextVersion. - models.UniqueConstraint( - fields=["learning_context_version_id", "item_version_id"], - name="lcviv_uniq_lcv_iv", - ), - - # An Item should have at most one version of itself published as - # part of any given LearningContextVersion. Having multiple - # ItemVersions from the same Item in a given LearningContextVersion - # would cause the identifiers to collide, which could cause buggy - # behavior without much benefit. - # - # Ideally, we could enforce this with a constraint condition that - # queried item_version.item, but MySQL does not support this. So we - # waste a little extra space to help enforce data integrity by - # adding a foreign key to the Item directly in this model, and then - # checking the uniqueness of (LearningContextVersion, Item). - models.UniqueConstraint( - fields=["learning_context_version", "item"], - name="lcviv_uniq_lcv_item" - ) - ] - - -class Content(models.Model): - """ - This is the most basic piece of raw content data, with no version metadata. - - Content stores data in an immutable Binary BLOB `data` field. This data is - not auto-normalized in any way, meaning that pieces of content that are - semantically equivalent (e.g. differently spaced/sorted JSON) will result in - new entries. This model is intentionally ignorant of what these things mean, - because it expects supplemental data models to build on top of it. - - Two Content instances _can_ have the same hash_digest if they are of - different MIME types. For instance, an empty text file and an empty SRT file - with both hash the same way, but be considered different entities. - - The other fields on Content are for data that is intrinsic to the file data - itself (e.g. the size). Any smart parsing of the contents into more - structured metadata should happen in other models that hang off of ItemInfo. - - Content models are not versioned in any way. The concept of versioning - exists at a higher level. - - Since this model uses a BinaryField to hold its data, we have to be careful - about scalability issues. For instance, video files should not be stored - here directly. There is a 10 MB limit set for the moment, to accomodate - things like PDF files and images, but the itention is for the vast majority - of rows to be much smaller than that. - """ - # Cap item size at 10 MB for now. - MAX_SIZE = 10_000_000 - - learning_context = models.ForeignKey(LearningContext, on_delete=models.CASCADE) - hash_digest = hash_field() - - # Per RFC 4288, MIME type and sub-type may each be 127 chars. - type = models.CharField(max_length=127, blank=False, null=False) - sub_type = models.CharField(max_length=127, blank=False, null=False) - - size = models.PositiveBigIntegerField( - validators=[MaxValueValidator(MAX_SIZE)], - ) - - # This should be manually set so that multiple Content rows being set in the - # same transaction are created with the same timestamp. The timestamp should - # be UTC. - created = manual_date_time_field() - - data = models.BinaryField(null=False, max_length=MAX_SIZE) - - def mime_type(self): - return f"{self.type}/{self.sub_type}" - - class Meta: - constraints = [ - # Make sure we don't store duplicates of this raw data within the - # same LearningContext, unless they're of different mime types. - models.UniqueConstraint( - fields=["learning_context", "type", "sub_type", "hash_digest"], - name="content_uniq_lc_hd", - ) - ] - -class ItemVersionContent(models.Model): - """ - Determines the Content for a given ItemVersion. - - An ItemVersion may be associated with multiple pieces of binary data. For - instance, a Video version might be associated with multiple transcripts in - different languages. - - When Content is associated with an ItemVersion, it has some local identifier - that is unique within the the context of that ItemVersion. This allows the - ItemVersion to do things like store an image file and reference it by a - "path" identifier. - - Content is immutable and sharable across multiple ItemVersions and even - across LearningContexts. - """ - item_version = models.ForeignKey(ItemVersion, on_delete=models.CASCADE) - content = models.ForeignKey(Content, on_delete=models.RESTRICT) - identifier = identifier_field() - - class Meta: - constraints = [ - # Uniqueness is only by ItemVersion and identifier. If for some - # reason an ItemVersion wants to associate the same piece of content - # with two different identifiers, that is permitted. - models.UniqueConstraint( - fields=["item_version", "identifier"], - name="itemversioncontent_uniq_iv_id", - ), - ] - indexes = [ - models.Index( - fields=['content', 'item_version'], - name="itemversioncontent_c_iv", - ), - models.Index( - fields=['item_version', 'content'], - name="itemversioncontent_iv_d", - ), - ] - diff --git a/openedx_learning/core/itemstore/models_api.py b/openedx_learning/core/itemstore/models_api.py deleted file mode 100644 index 83bfe006..00000000 --- a/openedx_learning/core/itemstore/models_api.py +++ /dev/null @@ -1,21 +0,0 @@ -from django.db import models - -from .models import ItemVersion - - -class ItemVersionDataMixin(models.Model): - """ - Minimal abstract model to let people attach data to ItemVersions. - - The idea is that if you have a model that is associated with a specific - version of an item, the join is going to be 1:1 with an ItemVersion, and - potentially M:1 with your data model. - """ - item_version = models.OneToOneField( - ItemVersion, - on_delete=models.CASCADE, - primary_key=True, - ) - - class Meta: - abstract = True diff --git a/openedx_learning/core/publishing/admin.py b/openedx_learning/core/publishing/admin.py index 23867af1..7040f900 100644 --- a/openedx_learning/core/publishing/admin.py +++ b/openedx_learning/core/publishing/admin.py @@ -1,19 +1,29 @@ from django.contrib import admin -from .models import ( - LearningContext, - LearningContextVersion, -) +from .models import LearningPackage, PublishLogEntry -# @admin.register(LearningContext) -# class LearningContextAdmin(admin.ModelAdmin): -# pass -admin.site.register(LearningContext) -admin.site.register(LearningContextVersion) +@admin.register(LearningPackage) +class LearningPackageAdmin(admin.ModelAdmin): + fields = ("identifier", "title", "uuid", "created", "updated") + readonly_fields = ("identifier", "title", "uuid", "created", "updated") + list_display = ("identifier", "title", "uuid", "created", "updated") -""" -admin.site.register(LearningContextBranch) -admin.site.register(LearningAppVersionReport) -admin.site.register(LearningAppContentError) -""" \ No newline at end of file + +@admin.register(PublishLogEntry) +class PublishLogEntryAdmin(admin.ModelAdmin): + fields = ("uuid", "learning_package", "published_at", "published_by", "message") + readonly_fields = ( + "uuid", + "learning_package", + "published_at", + "published_by", + "message", + ) + list_display = ( + "uuid", + "learning_package", + "published_at", + "published_by", + "message", + ) diff --git a/openedx_learning/core/publishing/api.py b/openedx_learning/core/publishing/api.py deleted file mode 100644 index 71533494..00000000 --- a/openedx_learning/core/publishing/api.py +++ /dev/null @@ -1,49 +0,0 @@ -""" -The order of data that must come in: - -* partitioning <-- multiple things will reference this for data integrity. -* policy, composition <-- navigation relies on composition -* navigation -* scheduling - -What does a publishing call look like, in a pluggable world? And how much data -are we talking about? - -Boundary between "composition" and "navigation" – fuzzy? Navigation has Unit -metadata, but doesn't know about anything _inside_ the Unit:: - - { - "type": "update", // as opposed to "replace" - "version": "someversionindicator", - - "policy": { - - }, - - "partitioning": { - - }, - "composition": { - - }, - "navigation": { - "type": "three_level_static", // This is a terrible name, what do we call what we have?, - - } - - } - -How to manage plugin cycle life? - -""" - - -def current_version(learning_context_key): - pass - - -def update_published_version(learning_context_key, app_name, published_at=None): - pass - - - diff --git a/openedx_learning/core/publishing/apps.py b/openedx_learning/core/publishing/apps.py index 362b5475..4a98507c 100644 --- a/openedx_learning/core/publishing/apps.py +++ b/openedx_learning/core/publishing/apps.py @@ -9,6 +9,7 @@ class PublishingConfig(AppConfig): """ Configuration for the publishing Django application. """ + name = "openedx_learning.core.publishing" verbose_name = "Learning Core: Publishing" - default_auto_field = 'django.db.models.BigAutoField' + default_auto_field = "django.db.models.BigAutoField" diff --git a/openedx_learning/core/publishing/migrations/0001_initial.py b/openedx_learning/core/publishing/migrations/0001_initial.py index cf3b38be..16f01183 100644 --- a/openedx_learning/core/publishing/migrations/0001_initial.py +++ b/openedx_learning/core/publishing/migrations/0001_initial.py @@ -1,5 +1,6 @@ -# Generated by Django 3.2.10 on 2022-07-21 04:01 +# Generated by Django 4.1 on 2023-02-10 18:56 +from django.conf import settings from django.db import migrations, models import django.db.models.deletion import uuid @@ -10,30 +11,89 @@ class Migration(migrations.Migration): initial = True dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), ] operations = [ migrations.CreateModel( - name='LearningContext', + name="LearningPackage", fields=[ - ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('uuid', models.UUIDField(default=uuid.uuid4, editable=False, unique=True)), - ('identifier', models.CharField(max_length=255)), - ('created', models.DateTimeField()), + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "uuid", + models.UUIDField( + default=uuid.uuid4, + editable=False, + unique=True, + verbose_name="UUID", + ), + ), + ("identifier", models.CharField(max_length=255)), + ("title", models.CharField(max_length=1000)), + ("created", models.DateTimeField()), + ("updated", models.DateTimeField()), ], + options={ + "verbose_name": "Learning Package", + "verbose_name_plural": "Learning Packages", + }, ), migrations.CreateModel( - name='LearningContextVersion', + name="PublishLogEntry", fields=[ - ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('uuid', models.UUIDField(default=uuid.uuid4, editable=False, unique=True)), - ('created', models.DateTimeField()), - ('learning_context', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='publishing.learningcontext')), - ('prev_version', models.ForeignKey(null=True, on_delete=django.db.models.deletion.RESTRICT, related_name='+', to='publishing.learningcontext')), + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "uuid", + models.UUIDField( + default=uuid.uuid4, + editable=False, + unique=True, + verbose_name="UUID", + ), + ), + ("message", models.CharField(blank=True, default="", max_length=1000)), + ("published_at", models.DateTimeField()), + ( + "learning_package", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="publishing.learningpackage", + ), + ), + ( + "published_by", + models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.SET_NULL, + to=settings.AUTH_USER_MODEL, + ), + ), ], + options={ + "verbose_name": "Publish Log Entry", + "verbose_name_plural": "Publish Log Entries", + }, ), migrations.AddConstraint( - model_name='learningcontext', - constraint=models.UniqueConstraint(fields=('identifier',), name='lc_uniq_identifier'), + model_name="learningpackage", + constraint=models.UniqueConstraint( + fields=("identifier",), name="lp_uniq_identifier" + ), ), ] diff --git a/openedx_learning/core/publishing/models.py b/openedx_learning/core/publishing/models.py index 19a4b643..293555ff 100644 --- a/openedx_learning/core/publishing/models.py +++ b/openedx_learning/core/publishing/models.py @@ -1,12 +1,6 @@ """ Idea: This app has _only_ things related to Publishing any kind of content -associated with a LearningContext. So in that sense: - -* LearningContext (might even go elsewhere) -* LearningContextVersion -* something to mark that an app has created a version for an LC -* something to handle errors -* a mixin for doing efficient version tracking +associated with a LearningPackage. """ from django.db import models from django.conf import settings @@ -18,57 +12,49 @@ ) -class LearningContext(models.Model): +class LearningPackage(models.Model): uuid = immutable_uuid_field() identifier = identifier_field() + title = models.CharField(max_length=1000, null=False, blank=False) + created = manual_date_time_field() + updated = manual_date_time_field() def __str__(self): - return f"LearningContext {self.uuid}: {self.identifier}" + return f"{self.identifier}: {self.title}" class Meta: constraints = [ - # LearningContext identifiers must be globally unique. This is + # LearningPackage identifiers must be globally unique. This is # something that might be relaxed in the future if this system were # to be extensible to something like multi-tenancy, in which case # we'd tie it to something like a Site or Org. - models.UniqueConstraint(fields=["identifier"], name="lc_uniq_identifier") + models.UniqueConstraint(fields=["identifier"], name="lp_uniq_identifier") ] - -class LearningContextVersion(models.Model): - """ - """ - uuid = immutable_uuid_field() - learning_context = models.ForeignKey(LearningContext, on_delete=models.CASCADE) - prev_version = models.ForeignKey(LearningContext, on_delete=models.RESTRICT, null=True, related_name='+') - created = manual_date_time_field() + verbose_name = "Learning Package" + verbose_name_plural = "Learning Packages" -# Placeholder: -""" -class PublishLog(models.Model): - learning_context = models.ForeignKey(LearningContext, on_delete=models.CASCADE) - version_num = models.PositiveIntegerField() +class PublishLogEntry(models.Model): + """ + This model tracks Publishing activity. - # Note: The same LearningContextVersion can show up multiple times if it's - # the case that something was reverted to an earlier version. - learning_context_version = models.ForeignKey(LearningContextVersion, on_delete=models.RESTRICT) + It is expected that other apps make foreign keys to this table to mark when + their content gets published. This is to allow us to tie together many + different entities (e.g. Components, Units, etc.) that are all published at + the same time. + """ + uuid = immutable_uuid_field() + learning_package = models.ForeignKey(LearningPackage, on_delete=models.CASCADE) + message = models.CharField(max_length=1000, null=False, blank=True, default="") published_at = manual_date_time_field() - published_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.SET_NULL, null=True) + published_by = models.ForeignKey( + settings.AUTH_USER_MODEL, + on_delete=models.SET_NULL, + null=True, + ) class Meta: - constraints = [ - # LearningContextVersions are created in a linear history, because - # every version is intended as either a published version or a draft - # version that is a candidate for publishing. In the event of a race - # condition of two processes that are trying to publish the "next" - # version, we should only allow one to win, and fail the other one - # so that it has to re-read and re-try (instead of silently - # overwriting). - models.UniqueConstraint( - fields=["learning_context_id", "version_num"], - name="learning_publish_pl_uniq_lc_vn", - ) - ] -""" + verbose_name = "Publish Log Entry" + verbose_name_plural = "Publish Log Entries" diff --git a/openedx_learning/core/publishing/readme.rst b/openedx_learning/core/publishing/readme.rst index c3dd1f32..44295dbd 100644 --- a/openedx_learning/core/publishing/readme.rst +++ b/openedx_learning/core/publishing/readme.rst @@ -6,7 +6,7 @@ The ``publishing`` app holds the core data models that allow different apps will Motivation ---------- -Content publishing is no longer a simple operation where one system processes a set of changes. The act of publishing in Open edX requires many systems to update their data, often through asynchronous tasks. Because each system is doing this independently, we can get into weird states where some systems have updated their data, others will do so in the following minutes, and some systems have failed entirely–for instance, course outlines might not match course contents, or search indexing +Content publishing is no longer a simple operation where one system processes a set of changes. The act of publishing in Open edX requires many systems to update their data, often through asynchronous tasks. Because each system is doing this independently, we can get into weird states where some systems have updated their data, others will do so in the following minutes, and some systems have failed entirely–for instance, course outlines might not match course contents, or search indexing. Intended Use Cases ------------------ @@ -14,7 +14,7 @@ Intended Use Cases * Create a new version of content for draft viewing purposes. * Publish a new version of content. -The idea is that a new LearningContextVersion is created, and an app builds all the data it needs for that version. Once all apps have built the necessary data, pointing the latest "published" version to be that version is a fast, atomic operation. +The idea is that a new LearningPackageVersion is created, and an app builds all the data it needs for that version. Once all apps have built the necessary data, pointing the latest "published" version to be that version is a fast, atomic operation. Architecture Guidelines diff --git a/openedx_learning/lib/fields.py b/openedx_learning/lib/fields.py index 6ff8c0d9..b18137c0 100644 --- a/openedx_learning/lib/fields.py +++ b/openedx_learning/lib/fields.py @@ -4,7 +4,7 @@ Field conventions: * Per OEP-38, we're using the MySQL-friendly convention of BigInt ID as a - primary key + separtate UUID column. + primary key + separate UUID column. https://open-edx-proposals.readthedocs.io/en/latest/best-practices/oep-0038-Data-Modeling.html TODO: @@ -28,7 +28,7 @@ def identifier_field(): Externally created Identifier fields. These will often be local to a particular scope, like within a - LearningContext. It's up to the application as to whether they're + LearningPackage. It's up to the application as to whether they're semantically meaningful or look more machine-generated. Other apps should *not* make references to these values directly, since @@ -40,6 +40,7 @@ def identifier_field(): null=False, ) + def immutable_uuid_field(): """ Stable, randomly-generated UUIDs. @@ -54,8 +55,10 @@ def immutable_uuid_field(): null=False, editable=False, unique=True, + verbose_name="UUID", # Just makes the Django admin output properly capitalized ) + def hash_field(): """ Holds a hash digest meant to identify a piece of content. @@ -74,9 +77,11 @@ def hash_field(): editable=False, ) + def create_hash_digest(data_bytes): return hashlib.blake2b(data_bytes, digest_size=20).hexdigest() + def manual_date_time_field(): """ DateTimeField that does not auto-generate values. diff --git a/openedx_learning/core/composition/__init__.py b/openedx_learning/rest_api/__init__.py similarity index 100% rename from openedx_learning/core/composition/__init__.py rename to openedx_learning/rest_api/__init__.py diff --git a/openedx_learning/rest_api/apps.py b/openedx_learning/rest_api/apps.py new file mode 100644 index 00000000..fe7f93f4 --- /dev/null +++ b/openedx_learning/rest_api/apps.py @@ -0,0 +1,11 @@ +from django.apps import AppConfig + + +class RESTAPIConfig(AppConfig): + """ + Configuration for the Learning Core REST API Django app. + """ + + name = "openedx_learning.rest_api" + verbose_name = "Learning Core: REST API" + default_auto_field = "django.db.models.BigAutoField" diff --git a/openedx_learning/rest_api/urls.py b/openedx_learning/rest_api/urls.py new file mode 100644 index 00000000..f6e1c2e8 --- /dev/null +++ b/openedx_learning/rest_api/urls.py @@ -0,0 +1,4 @@ +from django.contrib import admin +from django.urls import include, path + +urlpatterns = [path("v1/", include("openedx_learning.rest_api.v1.urls"))] diff --git a/openedx_learning/core/itemstore/__init__.py b/openedx_learning/rest_api/v1/__init__.py similarity index 100% rename from openedx_learning/core/itemstore/__init__.py rename to openedx_learning/rest_api/v1/__init__.py diff --git a/openedx_learning/rest_api/v1/components.py b/openedx_learning/rest_api/v1/components.py new file mode 100644 index 00000000..4fdc2261 --- /dev/null +++ b/openedx_learning/rest_api/v1/components.py @@ -0,0 +1,28 @@ +""" +This is just an example REST API endpoint. +""" +from rest_framework import viewsets +from rest_framework.response import Response + +from openedx_learning.core.components.models import Component + + +class ComponentViewSet(viewsets.ViewSet): + def list(self, request): + items = Component.objects.all() + raise NotImplementedError + + def retrieve(self, request, pk=None): + raise NotImplementedError + + def create(self, request): + raise NotImplementedError + + def update(self, request, pk=None): + raise NotImplementedError + + def partial_update(self, request, pk=None): + raise NotImplementedError + + def destroy(self, request, pk=None): + raise NotImplementedError diff --git a/openedx_learning/rest_api/v1/urls.py b/openedx_learning/rest_api/v1/urls.py new file mode 100644 index 00000000..3b1a66c3 --- /dev/null +++ b/openedx_learning/rest_api/v1/urls.py @@ -0,0 +1,7 @@ +from rest_framework.routers import DefaultRouter + +from . import components + +router = DefaultRouter() +router.register(r"components", components.ComponentViewSet, basename="component") +urlpatterns = router.urls diff --git a/projects/dev.py b/projects/dev.py index df59466f..13cb86c6 100644 --- a/projects/dev.py +++ b/projects/dev.py @@ -32,16 +32,18 @@ 'django.contrib.admindocs', # Learning Core Apps - 'openedx_learning.core.composition.apps.CompositionConfig', - 'openedx_learning.core.itemstore.apps.ItemStoreConfig', + 'openedx_learning.core.components.apps.ComponentsConfig', 'openedx_learning.core.publishing.apps.PublishingConfig', # Learning Contrib Apps - 'openedx_learning.contrib.staticassets.apps.StaticAssetsConfig', # Apps that don't belong in this repo in the long term, but are here to make # testing/iteration easier until the APIs stabilize. 'olx_importer.apps.OLXImporterConfig', + + # REST API + 'rest_framework', + 'openedx_learning.rest_api.apps.RESTAPIConfig', ) MIDDLEWARE = [ @@ -85,7 +87,7 @@ 'django.contrib.staticfiles.finders.AppDirectoriesFinder', ] STATICFILES_DIRS = [ - BASE_DIR / 'projects' / 'static' +# BASE_DIR / 'projects' / 'static' ] MEDIA_URL = '/media/' diff --git a/projects/urls.py b/projects/urls.py index 0cebc3f7..92928a09 100644 --- a/projects/urls.py +++ b/projects/urls.py @@ -4,4 +4,6 @@ urlpatterns = [ path('admin/doc/', include('django.contrib.admindocs.urls')), path('admin/', admin.site.urls), + + path('rest_api/', include('openedx_learning.rest_api.urls')) ] diff --git a/requirements/base.in b/requirements/base.in index 6a909522..c0122e9d 100644 --- a/requirements/base.in +++ b/requirements/base.in @@ -1,6 +1,15 @@ # Core requirements for using this application -c constraints.txt -Django<4.0 # Web application framework +Django # Web application framework +# For the Python API layer (eventually) attrs + +# Serialization +pyyaml + +# Django Rest Framework + extras for the openedx_lor project +djangorestframework +markdown +django-filter diff --git a/requirements/base.txt b/requirements/base.txt index 40f87f78..ff9722db 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -8,9 +8,26 @@ asgiref==3.5.2 # via django attrs==22.1.0 # via -r requirements/base.in -django==3.2.15 - # via -r requirements/base.in -pytz==2022.1 +backports-zoneinfo==0.2.1 # via django +django==4.1 + # via + # -r requirements/base.in + # django-filter + # djangorestframework +django-filter==22.1 + # via -r requirements/base.in +djangorestframework==3.13.1 + # via -r requirements/base.in +importlib-metadata==4.12.0 + # via markdown +markdown==3.4.1 + # via -r requirements/base.in +pytz==2022.2.1 + # via djangorestframework +pyyaml==6.0 + # via -r requirements/base.in sqlparse==0.4.2 # via django +zipp==3.8.1 + # via importlib-metadata diff --git a/requirements/ci.txt b/requirements/ci.txt index bd0b6ffa..912932df 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -6,27 +6,27 @@ # certifi==2022.6.15 # via requests -charset-normalizer==2.1.0 +charset-normalizer==2.1.1 # via requests click==8.1.3 # via import-linter codecov==2.1.12 # via -r requirements/ci.in -coverage==6.4.3 +coverage==6.4.4 # via codecov -distlib==0.3.5 +distlib==0.3.6 # via virtualenv -filelock==3.7.1 +filelock==3.8.0 # via # tox # virtualenv -grimp==1.2.3 +grimp==1.3 # via import-linter idna==3.3 # via requests -import-linter==1.2.7 +import-linter==1.3.0 # via -r requirements/ci.in -networkx==2.8.5 +networkx==2.8.6 # via grimp packaging==21.3 # via tox @@ -46,7 +46,9 @@ toml==0.10.2 # via tox tox==3.25.1 # via -r requirements/ci.in -urllib3==1.26.11 +typing-extensions==4.3.0 + # via import-linter +urllib3==1.26.12 # via requests -virtualenv==20.16.3 +virtualenv==20.16.4 # via tox diff --git a/requirements/dev.txt b/requirements/dev.txt index 33087f21..71421be7 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -8,7 +8,7 @@ asgiref==3.5.2 # via # -r requirements/quality.txt # django -astroid==2.11.7 +astroid==2.12.5 # via # -r requirements/quality.txt # pylint @@ -17,6 +17,10 @@ attrs==22.1.0 # via # -r requirements/quality.txt # pytest +backports-zoneinfo==0.2.1 + # via + # -r requirements/quality.txt + # django bleach==5.0.1 # via # -r requirements/quality.txt @@ -32,7 +36,7 @@ certifi==2022.6.15 # requests chardet==5.0.0 # via diff-cover -charset-normalizer==2.1.0 +charset-normalizer==2.1.1 # via # -r requirements/ci.txt # -r requirements/quality.txt @@ -61,7 +65,7 @@ commonmark==0.9.1 # via # -r requirements/quality.txt # rich -coverage[toml]==6.4.3 +coverage[toml]==6.4.4 # via # -r requirements/ci.txt # -r requirements/quality.txt @@ -73,14 +77,20 @@ dill==0.3.5.1 # via # -r requirements/quality.txt # pylint -distlib==0.3.5 +distlib==0.3.6 # via # -r requirements/ci.txt # virtualenv -django==3.2.15 +django==4.1 # via # -r requirements/quality.txt + # django-filter + # djangorestframework # edx-i18n-tools +django-filter==22.1 + # via -r requirements/quality.txt +djangorestframework==3.13.1 + # via -r requirements/quality.txt docutils==0.19 # via # -r requirements/quality.txt @@ -89,12 +99,12 @@ edx-i18n-tools==0.9.1 # via -r requirements/dev.in edx-lint==5.2.4 # via -r requirements/quality.txt -filelock==3.7.1 +filelock==3.8.0 # via # -r requirements/ci.txt # tox # virtualenv -grimp==1.2.3 +grimp==1.3 # via # -r requirements/ci.txt # import-linter @@ -103,12 +113,13 @@ idna==3.3 # -r requirements/ci.txt # -r requirements/quality.txt # requests -import-linter==1.2.7 +import-linter==1.3.0 # via -r requirements/ci.txt importlib-metadata==4.12.0 # via # -r requirements/quality.txt # keyring + # markdown # twine iniconfig==1.1.1 # via @@ -118,12 +129,16 @@ isort==5.10.1 # via # -r requirements/quality.txt # pylint +jaraco-classes==3.2.2 + # via + # -r requirements/quality.txt + # keyring jinja2==3.1.2 # via # -r requirements/quality.txt # code-annotations # diff-cover -keyring==23.8.1 +keyring==23.9.0 # via # -r requirements/quality.txt # twine @@ -131,6 +146,8 @@ lazy-object-proxy==1.7.1 # via # -r requirements/quality.txt # astroid +markdown==3.4.1 + # via -r requirements/quality.txt markupsafe==2.1.1 # via # -r requirements/quality.txt @@ -139,7 +156,11 @@ mccabe==0.7.0 # via # -r requirements/quality.txt # pylint -networkx==2.8.5 +more-itertools==8.14.0 + # via + # -r requirements/quality.txt + # jaraco-classes +networkx==2.8.6 # via # -r requirements/ci.txt # grimp @@ -153,7 +174,7 @@ packaging==21.3 # tox path==16.4.0 # via edx-i18n-tools -pbr==5.9.0 +pbr==5.10.0 # via # -r requirements/quality.txt # stevedore @@ -192,13 +213,13 @@ pycodestyle==2.9.1 # via -r requirements/quality.txt pydocstyle==6.1.1 # via -r requirements/quality.txt -pygments==2.12.0 +pygments==2.13.0 # via # -r requirements/quality.txt # diff-cover # readme-renderer # rich -pylint==2.14.5 +pylint==2.15.0 # via # -r requirements/quality.txt # edx-lint @@ -224,7 +245,7 @@ pyparsing==3.0.9 # -r requirements/pip-tools.txt # -r requirements/quality.txt # packaging -pytest==7.1.2 +pytest==7.1.3 # via # -r requirements/quality.txt # pytest-cov @@ -237,16 +258,16 @@ python-slugify==6.1.2 # via # -r requirements/quality.txt # code-annotations -pytz==2022.1 +pytz==2022.2.1 # via # -r requirements/quality.txt - # django + # djangorestframework pyyaml==6.0 # via # -r requirements/quality.txt # code-annotations # edx-i18n-tools -readme-renderer==36.0 +readme-renderer==37.0 # via # -r requirements/quality.txt # twine @@ -305,7 +326,7 @@ tomli==2.0.1 # pep517 # pylint # pytest -tomlkit==0.11.1 +tomlkit==0.11.4 # via # -r requirements/quality.txt # pylint @@ -319,17 +340,19 @@ twine==4.0.1 # via -r requirements/quality.txt typing-extensions==4.3.0 # via + # -r requirements/ci.txt # -r requirements/quality.txt # astroid + # import-linter # pylint # rich -urllib3==1.26.11 +urllib3==1.26.12 # via # -r requirements/ci.txt # -r requirements/quality.txt # requests # twine -virtualenv==20.16.3 +virtualenv==20.16.4 # via # -r requirements/ci.txt # tox diff --git a/requirements/doc.txt b/requirements/doc.txt index f2f1d75c..338ada5f 100644 --- a/requirements/doc.txt +++ b/requirements/doc.txt @@ -16,11 +16,15 @@ attrs==22.1.0 # pytest babel==2.10.3 # via sphinx +backports-zoneinfo==0.2.1 + # via + # -r requirements/test.txt + # django bleach==5.0.1 # via readme-renderer certifi==2022.6.15 # via requests -charset-normalizer==2.1.0 +charset-normalizer==2.1.1 # via requests click==8.1.3 # via @@ -28,11 +32,18 @@ click==8.1.3 # code-annotations code-annotations==1.3.0 # via -r requirements/test.txt -coverage[toml]==6.4.3 +coverage[toml]==6.4.4 # via # -r requirements/test.txt # pytest-cov -django==3.2.15 +django==4.1 + # via + # -r requirements/test.txt + # django-filter + # djangorestframework +django-filter==22.1 + # via -r requirements/test.txt +djangorestframework==3.13.1 # via -r requirements/test.txt doc8==1.0.0 # via -r requirements/doc.in @@ -49,7 +60,10 @@ idna==3.3 imagesize==1.4.1 # via sphinx importlib-metadata==4.12.0 - # via sphinx + # via + # -r requirements/test.txt + # markdown + # sphinx iniconfig==1.1.1 # via # -r requirements/test.txt @@ -59,6 +73,8 @@ jinja2==3.1.2 # -r requirements/test.txt # code-annotations # sphinx +markdown==3.4.1 + # via -r requirements/test.txt markupsafe==2.1.1 # via # -r requirements/test.txt @@ -68,7 +84,7 @@ packaging==21.3 # -r requirements/test.txt # pytest # sphinx -pbr==5.9.0 +pbr==5.10.0 # via # -r requirements/test.txt # stevedore @@ -80,7 +96,7 @@ py==1.11.0 # via # -r requirements/test.txt # pytest -pygments==2.12.0 +pygments==2.13.0 # via # doc8 # readme-renderer @@ -89,7 +105,7 @@ pyparsing==3.0.9 # via # -r requirements/test.txt # packaging -pytest==7.1.2 +pytest==7.1.3 # via # -r requirements/test.txt # pytest-cov @@ -102,16 +118,16 @@ python-slugify==6.1.2 # via # -r requirements/test.txt # code-annotations -pytz==2022.1 +pytz==2022.2.1 # via # -r requirements/test.txt # babel - # django + # djangorestframework pyyaml==6.0 # via # -r requirements/test.txt # code-annotations -readme-renderer==36.0 +readme-renderer==37.0 # via -r requirements/doc.in requests==2.28.1 # via sphinx @@ -160,9 +176,11 @@ tomli==2.0.1 # coverage # doc8 # pytest -urllib3==1.26.11 +urllib3==1.26.12 # via requests webencodings==0.5.1 # via bleach zipp==3.8.1 - # via importlib-metadata + # via + # -r requirements/test.txt + # importlib-metadata diff --git a/requirements/quality.txt b/requirements/quality.txt index 5b541835..615387d0 100644 --- a/requirements/quality.txt +++ b/requirements/quality.txt @@ -8,7 +8,7 @@ asgiref==3.5.2 # via # -r requirements/test.txt # django -astroid==2.11.7 +astroid==2.12.5 # via # pylint # pylint-celery @@ -16,11 +16,15 @@ attrs==22.1.0 # via # -r requirements/test.txt # pytest +backports-zoneinfo==0.2.1 + # via + # -r requirements/test.txt + # django bleach==5.0.1 # via readme-renderer certifi==2022.6.15 # via requests -charset-normalizer==2.1.0 +charset-normalizer==2.1.1 # via requests click==8.1.3 # via @@ -36,13 +40,20 @@ code-annotations==1.3.0 # edx-lint commonmark==0.9.1 # via rich -coverage[toml]==6.4.3 +coverage[toml]==6.4.4 # via # -r requirements/test.txt # pytest-cov dill==0.3.5.1 # via pylint -django==3.2.15 +django==4.1 + # via + # -r requirements/test.txt + # django-filter + # djangorestframework +django-filter==22.1 + # via -r requirements/test.txt +djangorestframework==3.13.1 # via -r requirements/test.txt docutils==0.19 # via readme-renderer @@ -52,7 +63,9 @@ idna==3.3 # via requests importlib-metadata==4.12.0 # via + # -r requirements/test.txt # keyring + # markdown # twine iniconfig==1.1.1 # via @@ -62,25 +75,31 @@ isort==5.10.1 # via # -r requirements/quality.in # pylint +jaraco-classes==3.2.2 + # via keyring jinja2==3.1.2 # via # -r requirements/test.txt # code-annotations -keyring==23.8.1 +keyring==23.9.0 # via twine lazy-object-proxy==1.7.1 # via astroid +markdown==3.4.1 + # via -r requirements/test.txt markupsafe==2.1.1 # via # -r requirements/test.txt # jinja2 mccabe==0.7.0 # via pylint +more-itertools==8.14.0 + # via jaraco-classes packaging==21.3 # via # -r requirements/test.txt # pytest -pbr==5.9.0 +pbr==5.10.0 # via # -r requirements/test.txt # stevedore @@ -100,11 +119,11 @@ pycodestyle==2.9.1 # via -r requirements/quality.in pydocstyle==6.1.1 # via -r requirements/quality.in -pygments==2.12.0 +pygments==2.13.0 # via # readme-renderer # rich -pylint==2.14.5 +pylint==2.15.0 # via # edx-lint # pylint-celery @@ -122,7 +141,7 @@ pyparsing==3.0.9 # via # -r requirements/test.txt # packaging -pytest==7.1.2 +pytest==7.1.3 # via # -r requirements/test.txt # pytest-cov @@ -135,15 +154,15 @@ python-slugify==6.1.2 # via # -r requirements/test.txt # code-annotations -pytz==2022.1 +pytz==2022.2.1 # via # -r requirements/test.txt - # django + # djangorestframework pyyaml==6.0 # via # -r requirements/test.txt # code-annotations -readme-renderer==36.0 +readme-renderer==37.0 # via twine requests==2.28.1 # via @@ -179,7 +198,7 @@ tomli==2.0.1 # coverage # pylint # pytest -tomlkit==0.11.1 +tomlkit==0.11.4 # via pylint twine==4.0.1 # via -r requirements/quality.in @@ -188,7 +207,7 @@ typing-extensions==4.3.0 # astroid # pylint # rich -urllib3==1.26.11 +urllib3==1.26.12 # via # requests # twine @@ -197,7 +216,6 @@ webencodings==0.5.1 wrapt==1.14.1 # via astroid zipp==3.8.1 - # via importlib-metadata - -# The following packages are considered to be unsafe in a requirements file: -# setuptools + # via + # -r requirements/test.txt + # importlib-metadata diff --git a/requirements/test.txt b/requirements/test.txt index 3146dfa4..a055df20 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -12,22 +12,39 @@ attrs==22.1.0 # via # -r requirements/base.txt # pytest +backports-zoneinfo==0.2.1 + # via + # -r requirements/base.txt + # django click==8.1.3 # via code-annotations code-annotations==1.3.0 # via -r requirements/test.in -coverage[toml]==6.4.3 +coverage[toml]==6.4.4 # via pytest-cov + # via + # -r requirements/base.txt + # django-filter + # djangorestframework +django-filter==22.1 + # via -r requirements/base.txt +djangorestframework==3.13.1 # via -r requirements/base.txt +importlib-metadata==4.12.0 + # via + # -r requirements/base.txt + # markdown iniconfig==1.1.1 # via pytest jinja2==3.1.2 # via code-annotations +markdown==3.4.1 + # via -r requirements/base.txt markupsafe==2.1.1 # via jinja2 packaging==21.3 # via pytest -pbr==5.9.0 +pbr==5.10.0 # via stevedore pluggy==1.0.0 # via pytest @@ -35,7 +52,7 @@ py==1.11.0 # via pytest pyparsing==3.0.9 # via packaging -pytest==7.1.2 +pytest==7.1.3 # via # pytest-cov # pytest-django @@ -45,12 +62,14 @@ pytest-django==4.5.2 # via -r requirements/test.in python-slugify==6.1.2 # via code-annotations -pytz==2022.1 +pytz==2022.2.1 # via # -r requirements/base.txt - # django + # djangorestframework pyyaml==6.0 - # via code-annotations + # via + # -r requirements/base.txt + # code-annotations sqlparse==0.4.2 # via # -r requirements/base.txt @@ -63,3 +82,7 @@ tomli==2.0.1 # via # coverage # pytest +zipp==3.8.1 + # via + # -r requirements/base.txt + # importlib-metadata diff --git a/test_settings.py b/test_settings.py index 6e2ccd35..e41be8f3 100644 --- a/test_settings.py +++ b/test_settings.py @@ -34,12 +34,12 @@ def root(*args): 'django.contrib.staticfiles', # Admin - 'django.contrib.admin', - 'django.contrib.admindocs', +# 'django.contrib.admin', +# 'django.contrib.admindocs', # Our own apps 'openedx_learning.core.publishing.apps.PublishingConfig', - 'openedx_learning.core.itemstore.apps.ItemStoreConfig', + 'openedx_learning.core.components.apps.ComponentsConfig', ] LOCALE_PATHS = [ diff --git a/tests/test_models.py b/tests/test_models.py index ed972f38..0e2faf6e 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -4,9 +4,9 @@ """ -class TestPublishedLearningContext: +class TestPublishedLearningPackage: """ - Tests of the PublishedLearningContext model. + Tests of the PublishedLearningPackage model. """ def test_something(self):