Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add ancestor annotation to kolibri_public export mapper. #4178

Merged
merged 1 commit into from
Jun 29, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions contentcuration/kolibri_public/tests/test_mapper.py
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,8 @@ def _assert_node(self, source, mapped):
for mapped_tag in mapped.tags.all():
self.assertEqual(OKAY_TAG, mapped_tag.tag_name)

self.assertEqual(mapped.ancestors, [{"id": ancestor.id, "title": ancestor.title} for ancestor in source.get_ancestors()])

def _recurse_and_assert(self, sources, mappeds, recursion_depth=0):
recursion_depths = []
for source, mapped in zip(sources, mappeds):
Expand Down
26 changes: 22 additions & 4 deletions contentcuration/kolibri_public/utils/mapper.py
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,8 @@ def run(self):
self.mapped_channel.public = self.public
self.mapped_channel.save_base(raw=True)
annotate_label_bitmasks(self.mapped_root.get_descendants(include_self=True))
# Rather than set the ancestors fields after mapping, like it is done in Kolibri
# here we set it during mapping as we are already recursing through the tree.
set_channel_metadata_fields(self.mapped_channel.id, public=self.public)

def _map_model(self, source, Model):
Expand All @@ -69,22 +71,30 @@ def _map_and_bulk_create_model(self, sources, Model):

Model.objects.bulk_create(cloned_sources, ignore_conflicts=True)

def _map_node(self, source):
return self._map_model(source, kolibri_public_models.ContentNode)
def _map_node(self, source, ancestors):
node = self._map_model(source, kolibri_public_models.ContentNode)
node.ancestors = ancestors
return node

def _extend_ancestors(self, ancestors, new_ancestor):
return ancestors + [{"id": new_ancestor.id, "title": new_ancestor.title.replace('"', '\\"')}]

def _recurse_to_create_tree(
self,
source,
nodes_by_parent,
ancestors,
):
nodes_to_create = [self._map_node(source)]
nodes_to_create = [self._map_node(source, ancestors)]

if source.kind == content_kinds.TOPIC and source.id in nodes_by_parent:
children = sorted(nodes_by_parent[source.id], key=lambda x: x.lft)
ancestors = self._extend_ancestors(ancestors, source)
for child in children:
nodes_to_create.extend(self._recurse_to_create_tree(
child,
nodes_by_parent,
ancestors,
))
return nodes_to_create

Expand All @@ -110,6 +120,7 @@ def _map(
self,
node,
batch_size,
ancestors=[],
progress_tracker=None,
):
"""
Expand All @@ -118,20 +129,24 @@ def _map(
if node.rght - node.lft < batch_size:
copied_nodes = self._deep_map(
node,
ancestors,
)
if progress_tracker:
progress_tracker.increment(len(copied_nodes))
return copied_nodes
node_copy = self._shallow_map(
node,
ancestors,
)
ancestors = self._extend_ancestors(ancestors, node)
if progress_tracker:
progress_tracker.increment()
children = node.get_children().order_by("lft")
for child in children:
self._map(
child,
batch_size,
ancestors=ancestors,
progress_tracker=progress_tracker,
)
return [node_copy]
Expand Down Expand Up @@ -189,8 +204,9 @@ def _copy_associated_objects(self, nodes):
def _shallow_map(
self,
node,
ancestors,
):
mapped_node = self._map_node(node)
mapped_node = self._map_node(node, ancestors)

mapped_node.save_base(raw=True)

Expand All @@ -200,6 +216,7 @@ def _shallow_map(
def _deep_map(
self,
node,
ancestors,
):
source_nodes = node.get_descendants(include_self=True)

Expand All @@ -212,6 +229,7 @@ def _deep_map(
nodes_to_create = self._recurse_to_create_tree(
node,
nodes_by_parent,
ancestors,
)

mapped_nodes = kolibri_public_models.ContentNode.objects.bulk_create(nodes_to_create)
Expand Down