From 86062a3acb36beea2001c81652805785635708c8 Mon Sep 17 00:00:00 2001 From: ozer550 Date: Fri, 8 Nov 2024 14:22:20 +0530 Subject: [PATCH 1/4] optimize garabge collection command --- contentcuration/contentcuration/models.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/contentcuration/contentcuration/models.py b/contentcuration/contentcuration/models.py index 690325fc3c..faa3ba243b 100644 --- a/contentcuration/contentcuration/models.py +++ b/contentcuration/contentcuration/models.py @@ -22,6 +22,7 @@ from django.core.mail import send_mail from django.core.validators import MaxValueValidator from django.core.validators import MinValueValidator +from django.db import connection from django.db import IntegrityError from django.db import models from django.db.models import Count @@ -261,9 +262,21 @@ def hard_delete_user_related_data(self): editable_channels_user_query, field="id")).filter(num_editors=1, public=False) # Point sole editor non-public channels' contentnodes to orphan tree to let - # our garbage collection delete the nodes and underlying files. - ContentNode._annotate_channel_id(ContentNode.objects).filter(channel_id__in=list( - non_public_channels_sole_editor.values_list("id", flat=True))).update(parent_id=settings.ORPHANAGE_ROOT_ID) + # our garbage collection delete the nodes and underlying file. + channel_tree_id_map = dict( + ContentNode.objects.values_list('id', 'main_tree__tree_id') + ) + + logging.debug("Queries after creating channel tree ID map: %s", connection.queries) + + tree_ids_to_update = [ + tree_id for id, tree_id in channel_tree_id_map.items() + if id in non_public_channels_sole_editor.values_list("id", flat=True) + ] + + ContentNode.objects.filter(tree_id__in=tree_ids_to_update).update(parent_id=settings.ORPHANAGE_ROOT_ID) + + logging.debug("Queries after updating content nodes parent ID: %s", connection.queries) # Hard delete non-public channels associated with this user (if user is the only editor). non_public_channels_sole_editor.delete() From d2e9353077adf3e74457e37373fd629cebe33ecc Mon Sep 17 00:00:00 2001 From: ozer550 Date: Fri, 8 Nov 2024 15:03:32 +0530 Subject: [PATCH 2/4] fix model in the query --- contentcuration/contentcuration/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contentcuration/contentcuration/models.py b/contentcuration/contentcuration/models.py index faa3ba243b..b5d55aa35c 100644 --- a/contentcuration/contentcuration/models.py +++ b/contentcuration/contentcuration/models.py @@ -264,7 +264,7 @@ def hard_delete_user_related_data(self): # Point sole editor non-public channels' contentnodes to orphan tree to let # our garbage collection delete the nodes and underlying file. channel_tree_id_map = dict( - ContentNode.objects.values_list('id', 'main_tree__tree_id') + Channel.objects.values_list('id', 'main_tree__tree_id') ) logging.debug("Queries after creating channel tree ID map: %s", connection.queries) From b849dbcb49ce5c2b41b2a189c5fc01cfff293160 Mon Sep 17 00:00:00 2001 From: ozer550 Date: Mon, 11 Nov 2024 15:01:09 +0530 Subject: [PATCH 3/4] indiviually loop through contentnodes for updates --- contentcuration/contentcuration/models.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/contentcuration/contentcuration/models.py b/contentcuration/contentcuration/models.py index b5d55aa35c..3d884bb2e4 100644 --- a/contentcuration/contentcuration/models.py +++ b/contentcuration/contentcuration/models.py @@ -269,12 +269,9 @@ def hard_delete_user_related_data(self): logging.debug("Queries after creating channel tree ID map: %s", connection.queries) - tree_ids_to_update = [ - tree_id for id, tree_id in channel_tree_id_map.items() - if id in non_public_channels_sole_editor.values_list("id", flat=True) - ] - - ContentNode.objects.filter(tree_id__in=tree_ids_to_update).update(parent_id=settings.ORPHANAGE_ROOT_ID) + for id, tree_id in channel_tree_id_map.items(): + if id in non_public_channels_sole_editor.values_list("id", flat=True): + ContentNode.objects.filter(tree_id=tree_id).update(parent_id=settings.ORPHANAGE_ROOT_ID) logging.debug("Queries after updating content nodes parent ID: %s", connection.queries) From 7038023e14f156e59dd0c22fa4cd1b234ce28567 Mon Sep 17 00:00:00 2001 From: ozer550 Date: Wed, 13 Nov 2024 15:15:39 +0530 Subject: [PATCH 4/4] avoid iterating over every channel --- contentcuration/contentcuration/models.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/contentcuration/contentcuration/models.py b/contentcuration/contentcuration/models.py index 3d884bb2e4..9c366ed787 100644 --- a/contentcuration/contentcuration/models.py +++ b/contentcuration/contentcuration/models.py @@ -263,15 +263,10 @@ def hard_delete_user_related_data(self): # Point sole editor non-public channels' contentnodes to orphan tree to let # our garbage collection delete the nodes and underlying file. - channel_tree_id_map = dict( - Channel.objects.values_list('id', 'main_tree__tree_id') - ) - - logging.debug("Queries after creating channel tree ID map: %s", connection.queries) + tree_ids_to_update = non_public_channels_sole_editor.values_list('main_tree__tree_id', flat=True) - for id, tree_id in channel_tree_id_map.items(): - if id in non_public_channels_sole_editor.values_list("id", flat=True): - ContentNode.objects.filter(tree_id=tree_id).update(parent_id=settings.ORPHANAGE_ROOT_ID) + for tree_id in tree_ids_to_update: + ContentNode.objects.filter(tree_id=tree_id).update(parent_id=settings.ORPHANAGE_ROOT_ID) logging.debug("Queries after updating content nodes parent ID: %s", connection.queries)