diff --git a/docs/CHANGELOG.rst b/docs/CHANGELOG.rst index f8364143f..30a1df33d 100644 --- a/docs/CHANGELOG.rst +++ b/docs/CHANGELOG.rst @@ -32,6 +32,9 @@ Changed authentication backend - Use ``keys`` instead of ``scan_iter`` in ``clear`` method of ``redis`` cache, since ``scan_iter`` is much slower +- Modified time propagates from ``Data`` object to ``Entity`` to ``Collection`` +- Modified time propagates from ``AnnotationValue`` to ``Entity`` +- Modified time propagates from ``Relation`` to ``Collection`` =================== diff --git a/resolwe/flow/migrations/0019_alter_annotationpreset_modified_and_more.py b/resolwe/flow/migrations/0019_alter_annotationpreset_modified_and_more.py new file mode 100644 index 000000000..16ce47a09 --- /dev/null +++ b/resolwe/flow/migrations/0019_alter_annotationpreset_modified_and_more.py @@ -0,0 +1,53 @@ +# Generated by Django 4.2.7 on 2023-12-06 11:59 + +from django.db import migrations +import resolwe.flow.models.base + + +class Migration(migrations.Migration): + dependencies = [ + ("flow", "0018_add_annotationvalue_default_order"), + ] + + operations = [ + migrations.AlterField( + model_name="annotationpreset", + name="modified", + field=resolwe.flow.models.base.ModifiedField(auto_now=True, db_index=True), + ), + migrations.AlterField( + model_name="collection", + name="modified", + field=resolwe.flow.models.base.ModifiedField(auto_now=True, db_index=True), + ), + migrations.AlterField( + model_name="data", + name="modified", + field=resolwe.flow.models.base.ModifiedField(auto_now=True, db_index=True), + ), + migrations.AlterField( + model_name="descriptorschema", + name="modified", + field=resolwe.flow.models.base.ModifiedField(auto_now=True, db_index=True), + ), + migrations.AlterField( + model_name="entity", + name="modified", + field=resolwe.flow.models.base.ModifiedField(auto_now=True, db_index=True), + ), + migrations.AlterField( + model_name="process", + name="modified", + field=resolwe.flow.models.base.ModifiedField(auto_now=True, db_index=True), + ), + migrations.AlterField( + model_name="relation", + name="modified", + field=resolwe.flow.models.base.ModifiedField(auto_now=True, db_index=True), + ), + migrations.AlterField( + model_name="storage", + name="modified", + field=resolwe.flow.models.base.ModifiedField(auto_now=True, db_index=True), + ), + ] diff --git a/resolwe/flow/models/annotations.py b/resolwe/flow/models/annotations.py index 2b7475530..8a5ce6385 100644 --- a/resolwe/flow/models/annotations.py +++ b/resolwe/flow/models/annotations.py @@ -18,6 +18,7 @@ from django.core.exceptions import ValidationError from django.db import models +from django.utils.timezone import now from resolwe.flow.models.base import BaseModel from resolwe.permissions.models import Permission, PermissionObject @@ -502,6 +503,8 @@ def save(self, *args, **kwargs): if "label" not in self._value: self.recompute_label() super().save(*args, **kwargs) + self.entity.modified = now() + self.entity.save(update_fields=["modified"]) def recompute_label(self): """Recompute label from value and set it to the model instance.""" diff --git a/resolwe/flow/models/base.py b/resolwe/flow/models/base.py index bbbb7a9bc..52416681a 100644 --- a/resolwe/flow/models/base.py +++ b/resolwe/flow/models/base.py @@ -16,6 +16,24 @@ MAX_NAME_LENGTH = 100 +class ModifiedField(models.DateTimeField): + """Custom datetime field. + + When attribute 'skip_auto_now' is set to True, the field is not updated in the + pre_save method even if auto_now is set to True. The 'skip_auto_now' is reset + on every save to the default value 'False'. + """ + + def pre_save(self, model_instance, add): + """Prepare field for saving.""" + # When skip_auto_now is set do not generate the new value even is auto_now is + # set to True. + if getattr(model_instance, "skip_auto_now", False) is True: + return getattr(model_instance, self.attname) + else: + return super().pre_save(model_instance, add) + + class UniqueSlugError(IntegrityError): """The error indicates slug collision.""" @@ -85,7 +103,7 @@ class Meta: created = models.DateTimeField(auto_now_add=True, db_index=True) #: modified date and time - modified = models.DateTimeField(auto_now=True, db_index=True) + modified = ModifiedField(auto_now=True, db_index=True) #: user that created the entry contributor = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.PROTECT) diff --git a/resolwe/flow/models/data.py b/resolwe/flow/models/data.py index 77351a3ae..8b6e8defe 100644 --- a/resolwe/flow/models/data.py +++ b/resolwe/flow/models/data.py @@ -570,6 +570,16 @@ def save(self, render_name=False, *args, **kwargs): with transaction.atomic(): self._perform_save(*args, **kwargs) + if self.entity: + self.entity.skip_auto_now = True # type: ignore + self.entity.modified = self.modified + self.entity.save(update_fields=["modified"]) + self.entity.skip_auto_now = False # type: ignore + elif self.collection: + self.collection.skip_auto_now = True # type: ignore + self.collection.modified = self.modified + self.collection.save(update_fields=["modified"]) + self.collection.skip_auto_now = False # type: ignore self._original_output = self.output @@ -621,7 +631,7 @@ def move_to_collection(self, collection): self.permission_group = collection.permission_group if collection: self.tags = collection.tags - self.save(update_fields=["tags", "permission_group", "collection"]) + self.save(update_fields=["tags", "permission_group", "collection", "modified"]) @move_to_container def move_to_entity(self, entity): @@ -635,7 +645,7 @@ def move_to_entity(self, entity): else: self.permission_group = entity.permission_group self.tags = entity.tags - self.save(update_fields=["permission_group", "entity", "tags"]) + self.save(update_fields=["permission_group", "entity", "tags", "modified"]) def validate_change_collection(self, collection): """Raise validation error if data object cannot change collection.""" diff --git a/resolwe/flow/models/entity.py b/resolwe/flow/models/entity.py index 2431a5228..dacae39a0 100644 --- a/resolwe/flow/models/entity.py +++ b/resolwe/flow/models/entity.py @@ -377,6 +377,15 @@ def validate_annotations(self): if validation_errors: raise ValidationError(validation_errors) + def save(self, *args, **kwargs): + """Propagate the modified time to the collection.""" + super().save(*args, **kwargs) + if self.collection: + self.collection.skip_auto_now = True # type: ignore + self.collection.modified = self.modified + self.collection.save(update_fields=["modified"]) + self.collection.skip_auto_now = False # type: ignore + class RelationType(models.Model): """Model for storing relation types.""" @@ -501,6 +510,11 @@ def save(self, *args, **kwargs): "`descriptor_schema` must be defined if `descriptor` is given" ) super().save() + if self.collection: + self.collection.skip_auto_now = True # type: ignore + self.collection.modified = self.modified + self.collection.save(update_fields=["modified"]) + self.collection.skip_auto_now = False # type: ignore class RelationPartition(models.Model): diff --git a/resolwe/flow/tests/test_annotations.py b/resolwe/flow/tests/test_annotations.py index cd4bc74ed..3dac49d1c 100644 --- a/resolwe/flow/tests/test_annotations.py +++ b/resolwe/flow/tests/test_annotations.py @@ -1,4 +1,5 @@ # pylint: disable=missing-docstring +import time from typing import Any, Sequence from django.core.exceptions import ValidationError @@ -164,6 +165,34 @@ def test_all_fields_included(self): self.assertIn(method_name, dir(self)) +class TestModifiedPropagated(TestCase): + def test_modified_propagated(self): + """Test modify propagates to entity.""" + entity = Entity.objects.create(contributor=self.contributor) + annotation_group = AnnotationGroup.objects.create( + name="group", label="Annotation group", sort_order=1 + ) + field = AnnotationField.objects.create( + name="field name", + label="field label", + type=AnnotationType.STRING.value, + sort_order=1, + group=annotation_group, + ) + + old_modified = entity.modified + time.sleep(0.1) + value = AnnotationValue.objects.create(entity=entity, value="Test", field=field) + entity.refresh_from_db() + self.assertGreater(entity.modified, old_modified) + old_modified = entity.modified + time.sleep(0.1) + value.value = "New value" + value.save() + entity.refresh_from_db() + self.assertGreater(entity.modified, old_modified) + + class TestOrderEntityByAnnotations(TestCase): """Test ordering Entities by annotation values.""" diff --git a/resolwe/flow/tests/test_models.py b/resolwe/flow/tests/test_models.py index 01795739c..6c95de659 100644 --- a/resolwe/flow/tests/test_models.py +++ b/resolwe/flow/tests/test_models.py @@ -1,6 +1,7 @@ # pylint: disable=missing-docstring,too-many-lines import os import shutil +import time from datetime import timedelta from unittest.mock import MagicMock, PropertyMock, patch @@ -310,6 +311,39 @@ def test_dependencies_list(self): {d.kind for d in third.parents_dependency.all()}, {DataDependency.KIND_IO} ) + def test_modified_propagated(self): + """Test modified field is propagated to the entity and collection.""" + process = Process.objects.create(contributor=self.contributor) + data = Data.objects.create(contributor=self.contributor, process=process) + collection = Collection.objects.create(contributor=self.contributor) + entity = Entity.objects.create( + contributor=self.contributor, collection=collection + ) + old_modified = data.modified + old_collection_modified = collection.modified + data.move_to_collection(collection) + time.sleep(0.1) + data.refresh_from_db() + collection.refresh_from_db() + self.assertGreater(data.modified, old_modified) + self.assertGreater(data.modified, old_collection_modified) + self.assertEqual(collection.modified, data.modified) + + data = Data.objects.create(contributor=self.contributor, process=process) + old_modified = data.modified + old_entity_modified = entity.modified + old_collection_modified = collection.modified + time.sleep(0.1) + data.move_to_entity(entity) + data.refresh_from_db() + collection.refresh_from_db() + entity.refresh_from_db() + self.assertGreater(data.modified, old_modified) + self.assertGreater(data.modified, old_collection_modified) + self.assertGreater(data.modified, old_entity_modified) + self.assertEqual(collection.modified, data.modified) + self.assertEqual(entity.modified, data.modified) + class EntityModelTest(TestCase): def setUp(self): @@ -330,6 +364,21 @@ def setUp(self): name="Test data", contributor=self.contributor, process=self.process ) + def test_modified_propagated(self): + """Test modified field is propagated to the entity and collection.""" + collection = Collection.objects.create(contributor=self.contributor) + entity = Entity.objects.create( + contributor=self.contributor, collection=collection + ) + old_entity_modified = entity.modified + entity.name = "New name" + time.sleep(0.1) + entity.save() + entity.refresh_from_db() + collection.refresh_from_db() + self.assertGreater(entity.modified, old_entity_modified) + self.assertEqual(collection.modified, entity.modified) + def test_delete_data(self): # Create another Data object and add it to the same Entity. data_2 = Data.objects.create( @@ -1728,3 +1777,5 @@ def test_referenced_mix(self): refs.remove("jsonout.txt") refs.remove("stdout.txt") self.assertSetEqual(set(refs), {"file1", "file2"}) + self.assertSetEqual(set(refs), {"file1", "file2"}) + self.assertSetEqual(set(refs), {"file1", "file2"}) diff --git a/resolwe/flow/tests/test_relations.py b/resolwe/flow/tests/test_relations.py index fb1989b5e..c6a72dc84 100644 --- a/resolwe/flow/tests/test_relations.py +++ b/resolwe/flow/tests/test_relations.py @@ -1,5 +1,6 @@ # pylint: disable=missing-docstring import os +import time from django.apps import apps from django.contrib.auth.models import AnonymousUser @@ -122,6 +123,17 @@ def setUp(self): ) self.collection.set_permission(Permission.VIEW, AnonymousUser()) + def test_modified_propagated(self): + """Test modified date propagetes to collection.""" + old_modified = self.collection.modified + time.sleep(0.1) + self.relation_group.contributor = self.user + self.relation_group.save() + self.collection.refresh_from_db() + self.relation_group.refresh_from_db() + self.assertGreater(self.collection.modified, old_modified) + self.assertEqual(self.relation_group.modified, self.collection.modified) + def test_prefetch(self): self.relation_group.delete() self.relation_series.delete() diff --git a/resolwe/observers/tests.py b/resolwe/observers/tests.py index 4fe8bab35..b59a17eb4 100644 --- a/resolwe/observers/tests.py +++ b/resolwe/observers/tests.py @@ -344,27 +344,45 @@ def update_data(): await update_data() - updates = [json.loads(await client.receive_from()) for _ in range(3)] + updates = [json.loads(await client.receive_from()) for _ in range(6)] self.assertCountEqual( updates, [ { "change_type": ChangeType.UPDATE.name, - "object_id": 40, + "object_id": data.collection.pk, "subscription_id": self.subscription_id.hex, - "source": ["data", 42], + "source": ["data", data.pk], }, { "change_type": ChangeType.UPDATE.name, - "object_id": 41, + "object_id": data.entity.pk, "subscription_id": self.subscription_id.hex, - "source": ["data", 42], + "source": ["data", data.pk], }, { "change_type": ChangeType.UPDATE.name, - "object_id": 42, + "object_id": data.pk, "subscription_id": self.subscription_id.hex, - "source": ["data", 42], + "source": ["data", data.pk], + }, + { + "change_type": ChangeType.UPDATE.name, + "object_id": data.entity.pk, + "subscription_id": self.subscription_id.hex, + "source": ["entity", data.entity.pk], + }, + { + "change_type": ChangeType.UPDATE.name, + "object_id": data.collection.pk, + "subscription_id": self.subscription_id.hex, + "source": ["entity", data.entity.pk], + }, + { + "change_type": ChangeType.UPDATE.name, + "object_id": data.collection.pk, + "subscription_id": self.subscription_id.hex, + "source": ["collection", data.collection.pk], }, ], ) @@ -376,21 +394,27 @@ def update_entity(): entity.save() await update_entity() - updates = [json.loads(await client.receive_from()) for _ in range(2)] + updates = [json.loads(await client.receive_from()) for _ in range(3)] self.assertCountEqual( updates, [ { "change_type": ChangeType.UPDATE.name, - "object_id": 40, + "object_id": data.collection.pk, + "subscription_id": self.subscription_id.hex, + "source": ["entity", data.entity.pk], + }, + { + "change_type": ChangeType.UPDATE.name, + "object_id": data.entity.pk, "subscription_id": self.subscription_id.hex, - "source": ["entity", 41], + "source": ["entity", data.entity.pk], }, { "change_type": ChangeType.UPDATE.name, - "object_id": 41, + "object_id": data.collection.pk, "subscription_id": self.subscription_id.hex, - "source": ["entity", 41], + "source": ["collection", data.collection.pk], }, ], ) @@ -1279,7 +1303,7 @@ def change_permission_group(data): json.loads(await client_bob.receive_from()) for _ in range(2) ] notifications_alice = [ - json.loads(await client_alice.receive_from()) for _ in range(3) + json.loads(await client_alice.receive_from()) for _ in range(4) ] # Assert that Bob sees this as a deletion. @@ -1303,6 +1327,12 @@ def change_permission_group(data): self.assertCountEqual( notifications_alice, [ + { + "object_id": self.collection2.pk, + "change_type": ChangeType.UPDATE.name, + "subscription_id": self.subscription_id2.hex, + "source": ["collection", self.collection2.pk], + }, { "object_id": data.pk, "change_type": ChangeType.UPDATE.name,