From 3f73db31d1d964d37998bb4af1321f19e6c59aa2 Mon Sep 17 00:00:00 2001 From: Jihoon Baek Date: Fri, 24 Apr 2020 15:45:50 -0400 Subject: [PATCH] Add bulk_update_with_history (#650) * Add bulk_update_with_history feature Using Django 2.2's bulk_update feature. This is significantly simpler than bulk_create_with_history because we don't have to worry about the primary key issue. * Add support for bulk_update_with_history * Added to the docstring. * initial tests for bulk_update_with_history * Update documentation for bulk_update_with_history * run make format * fix trailing whitespace * update this PR so that it's still compatible with django versions 1.11, 2.0 and 2.1 --- CHANGES.rst | 4 +++ docs/common_issues.rst | 28 ++++++++++++--- simple_history/manager.py | 14 ++++++-- simple_history/tests/tests/test_manager.py | 41 ++++++++++++++++++++++ simple_history/utils.py | 21 +++++++++++ 5 files changed, 101 insertions(+), 7 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index eae196a9c..899406e33 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,10 @@ Changes ======= +Unreleased +------------ +- Added `bulk_update_with_history` utility function (gh-650) + 2.9.0 (2020-04-23) ------------------ - Add simple filtering if provided a minutes argument in `clean_duplicate_history` (gh-606) diff --git a/docs/common_issues.rst b/docs/common_issues.rst index 72ff075d4..a14ca4290 100644 --- a/docs/common_issues.rst +++ b/docs/common_issues.rst @@ -5,7 +5,7 @@ Bulk Creating and Queryset Updating ----------------------------------- ``django-simple-history`` functions by saving history using a ``post_save`` signal every time that an object with history is saved. However, for certain bulk -operations, such as bulk_create_ and `queryset updates `_, +operations, such as bulk_create_, bulk_update_, and `queryset updates`_, signals are not sent, and the history is not saved automatically. However, ``django-simple-history`` provides utility functions to work around this. @@ -16,6 +16,7 @@ As of ``django-simple-history`` 2.2.0, we can use the utility function history: .. _bulk_create: https://docs.djangoproject.com/en/2.0/ref/models/querysets/#bulk-create +.. _bulk_update: https://docs.djangoproject.com/en/3.0/ref/models/querysets/#bulk-update .. code-block:: pycon @@ -42,9 +43,27 @@ can add `changeReason` on each instance: >>> Poll.history.get(id=data[0].id).history_change_reason 'reason' +Bulk Updating a Model with History (New) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Bulk update was introduced with Django 2.2. We can use the utility function +``bulk_update_with_history`` in order to bulk update objects using Django's ``bulk_update`` function while saving the object history: -QuerySet Updates with History -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: pycon + + >>> from simple_history.utils import bulk_create_with_history + >>> from simple_history.tests.models import Poll + >>> from django.utils.timezone import now + >>> + >>> data = [Poll(id=x, question='Question ' + str(x), pub_date=now()) for x in range(1000)] + >>> objs = bulk_create_with_history(data, Poll, batch_size=500) + >>> for obj in objs: obj.question = 'Duplicate Questions' + >>> bulk_update_with_history(objs, Poll, ['question'], batch_size=500) + >>> Poll.objects.first().question + 'Duplicate Question' + +QuerySet Updates with History (Updated in Django 2.2) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Unlike with ``bulk_create``, `queryset updates`_ perform an SQL update query on the queryset, and never return the actual updated objects (which would be necessary for the inserts into the historical table). Thus, we tell you that @@ -60,8 +79,9 @@ As the Django documentation says:: e.comments_on = False e.save() -.. _queryset updates: https://docs.djangoproject.com/en/2.0/ref/models/querysets/#update +.. _queryset updates: https://docs.djangoproject.com/en/2.2/ref/models/querysets/#update +Note: Django 2.2 now allows ``bulk_update``. No ``pre_save`` or ``post_save`` signals are sent still. Tracking Custom Users --------------------- diff --git a/simple_history/manager.py b/simple_history/manager.py index 0f3b957c8..2c20fdc35 100755 --- a/simple_history/manager.py +++ b/simple_history/manager.py @@ -98,8 +98,16 @@ def _as_of_set(self, date): continue yield last_change.instance - def bulk_history_create(self, objs, batch_size=None): - """Bulk create the history for the objects specified by objs""" + def bulk_history_create(self, objs, batch_size=None, update=False): + """ + Bulk create the history for the objects specified by objs. + If called by bulk_update_with_history, use the update boolean and + save the history_type accordingly. + """ + + history_type = "+" + if update: + history_type = "~" historical_instances = [] for instance in objs: @@ -107,7 +115,7 @@ def bulk_history_create(self, objs, batch_size=None): history_date=getattr(instance, "_history_date", timezone.now()), history_user=getattr(instance, "_history_user", None), history_change_reason=getattr(instance, "changeReason", ""), - history_type="+", + history_type=history_type, **{ field.attname: getattr(instance, field.attname) for field in instance._meta.fields diff --git a/simple_history/tests/tests/test_manager.py b/simple_history/tests/tests/test_manager.py index ecb8a5249..9ef94060d 100644 --- a/simple_history/tests/tests/test_manager.py +++ b/simple_history/tests/tests/test_manager.py @@ -158,3 +158,44 @@ def test_set_custom_history_user_on_first_obj(self): def test_efficiency(self): with self.assertNumQueries(1): Poll.history.bulk_history_create(self.data) + + +class BulkHistoryUpdateTestCase(TestCase): + def setUp(self): + self.data = [ + Poll(id=1, question="Question 1", pub_date=datetime.now()), + Poll(id=2, question="Question 2", pub_date=datetime.now()), + Poll(id=3, question="Question 3", pub_date=datetime.now()), + Poll(id=4, question="Question 4", pub_date=datetime.now()), + ] + + def test_simple_bulk_history_create(self): + created = Poll.history.bulk_history_create(self.data, update=True) + self.assertEqual(len(created), 4) + self.assertQuerysetEqual( + Poll.history.order_by("question"), + ["Question 1", "Question 2", "Question 3", "Question 4"], + attrgetter("question"), + ) + self.assertTrue( + all([history.history_type == "~" for history in Poll.history.all()]) + ) + + created = Poll.history.bulk_create([]) + self.assertEqual(created, []) + self.assertEqual(Poll.history.count(), 4) + + def test_bulk_history_create_with_change_reason(self): + for poll in self.data: + poll.changeReason = "reason" + + Poll.history.bulk_history_create(self.data) + + self.assertTrue( + all( + [ + history.history_change_reason == "reason" + for history in Poll.history.all() + ] + ) + ) diff --git a/simple_history/utils.py b/simple_history/utils.py index c8ed4a4e7..e0894fc2f 100644 --- a/simple_history/utils.py +++ b/simple_history/utils.py @@ -1,3 +1,4 @@ +import django from django.db import transaction from django.forms.models import model_to_dict @@ -72,3 +73,23 @@ def bulk_create_with_history(objs, model, batch_size=None): history_manager.bulk_history_create(obj_list, batch_size=batch_size) objs_with_id = obj_list return objs_with_id + + +def bulk_update_with_history(objs, model, fields, batch_size=None): + """ + Bulk update the objects specified by objs while also bulk creating + their history (all in one transaction). + :param objs: List of objs of type model to be updated + :param model: Model class that should be updated + :param fields: The fields that are updated + :param batch_size: Number of objects that should be updated in each batch + """ + if django.VERSION < (2, 2,): + raise NotImplementedError( + "bulk_update_with_history is only available on " + "Django versions 2.2 and later" + ) + history_manager = get_history_manager_for_model(model) + with transaction.atomic(savepoint=False): + model.objects.bulk_update(objs, fields, batch_size=batch_size) + history_manager.bulk_history_create(objs, batch_size=batch_size, update=True)