Merge pull request #2010 from dandi/asset-access-metadata

Derive asset `access` field from asset blob
dandi · Oct 17, 2024 · da7b975 · da7b975
2 parents 6d91584 + b5c0c0a
commit da7b975
Show file tree

Hide file tree

Showing 3 changed files with 102 additions and 0 deletions.
diff --git a/dandiapi/api/migrations/0011_asset_access_metadata.py b/dandiapi/api/migrations/0011_asset_access_metadata.py
@@ -0,0 +1,81 @@
+# Generated by Django 4.1.13 on 2024-08-20 17:21
+from __future__ import annotations
+
+from django.db import migrations, models
+from django.db.models.expressions import RawSQL
+
+
+def remove_access_fields(apps, _):
+    Asset = apps.get_model('api.Asset')
+    AuditRecord = apps.get_model('api.AuditRecord')
+
+    # Use the postgres jsonb '-' operator to delete the 'access' field from metadata
+    Asset.objects.filter(published=False, metadata__access__isnull=False).update(
+        metadata=RawSQL("metadata - 'access'", [])
+    )
+
+    # Delete access field from existing audit records
+    # https://www.postgresql.org/docs/current/functions-json.html#:~:text=jsonb%20%23%2D%20text%5B%5D%20%E2%86%92%20jsonb
+    AuditRecord.objects.filter(record_type__in=['add_asset', 'update_asset']).update(
+        details=RawSQL("details #- '{metadata, access}'", [])
+    )
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ('api', '0010_auditrecord'),
+    ]
+
+    operations = [
+        migrations.RunPython(remove_access_fields),
+        migrations.RemoveConstraint(
+            model_name='asset',
+            name='asset_metadata_no_computed_keys_or_published',
+        ),
+        migrations.AddConstraint(
+            model_name='asset',
+            constraint=models.CheckConstraint(
+                check=models.Q(
+                    models.Q(
+                        ('published', False),
+                        models.Q(
+                            (
+                                'metadata__has_any_keys',
+                                [
+                                    'id',
+                                    'access',
+                                    'path',
+                                    'identifier',
+                                    'contentUrl',
+                                    'contentSize',
+                                    'digest',
+                                    'datePublished',
+                                    'publishedBy',
+                                ],
+                            ),
+                            _negated=True,
+                        ),
+                    ),
+                    models.Q(
+                        ('published', True),
+                        (
+                            'metadata__has_keys',
+                            [
+                                'id',
+                                'access',
+                                'path',
+                                'identifier',
+                                'contentUrl',
+                                'contentSize',
+                                'digest',
+                                'datePublished',
+                                'publishedBy',
+                            ],
+                        ),
+                    ),
+                    _connector='OR',
+                ),
+                name='asset_metadata_no_computed_keys_or_published',
+            ),
+        ),
+    ]
diff --git a/dandiapi/api/models/asset.py b/dandiapi/api/models/asset.py
@@ -6,6 +6,7 @@
 from urllib.parse import urlparse, urlunparse
 import uuid
 
+from dandischema.models import AccessType
 from django.conf import settings
 from django.contrib.postgres.indexes import HashIndex
 from django.core.exceptions import ValidationError
@@ -24,6 +25,7 @@
 ASSET_PATH_REGEX = rf'^({ASSET_CHARS_REGEX}?\/?\.?{ASSET_CHARS_REGEX})+$'
 ASSET_COMPUTED_FIELDS = [
     'id',
+    'access',
     'path',
     'identifier',
     'contentUrl',
@@ -236,6 +238,15 @@ def full_metadata(self):
         metadata = {
             **self.metadata,
             'id': self.dandi_asset_id(self.asset_id),
+            'access': [
+                {
+                    'schemaKey': 'AccessRequirements',
+                    # TODO: When embargoed zarrs land, include that logic here
+                    'status': AccessType.EmbargoedAccess.value
+                    if self.blob and self.blob.embargoed
+                    else AccessType.OpenAccess.value,
+                }
+            ],
             'path': self.path,
             'identifier': str(self.asset_id),
             'contentUrl': [download_url, self.s3_url],

diff --git a/dandiapi/api/tests/test_asset.py b/dandiapi/api/tests/test_asset.py
@@ -3,6 +3,7 @@
 import json
 from uuid import uuid4
 
+from dandischema.models import AccessType
 from django.conf import settings
 from django.db.utils import IntegrityError
 from django.urls import reverse
@@ -193,6 +194,7 @@ def test_asset_full_metadata(draft_asset_factory):
     assert asset.full_metadata == {
         **raw_metadata,
         'id': f'dandiasset:{asset.asset_id}',
+        'access': [{'schemaKey': 'AccessRequirements', 'status': AccessType.OpenAccess.value}],
         'path': asset.path,
         'identifier': str(asset.asset_id),
         'contentUrl': [download_url, blob_url],
@@ -219,6 +221,7 @@ def test_asset_full_metadata_zarr(draft_asset_factory, zarr_archive):
     assert asset.full_metadata == {
         **raw_metadata,
         'id': f'dandiasset:{asset.asset_id}',
+        'access': [{'schemaKey': 'AccessRequirements', 'status': AccessType.OpenAccess.value}],
         'path': asset.path,
         'identifier': str(asset.asset_id),
         'contentUrl': [download_url, s3_url],
@@ -679,6 +682,12 @@ def test_asset_create_embargo(
         'meta': 'data',
         'foo': ['bar', 'baz'],
         '1': 2,
+        'access': [
+            {
+                'schemaKey': 'AccessRequirements',
+                'status': AccessType.OpenAccess.value,
+            }
+        ],
     }
 
     resp = api_client.post(
@@ -689,6 +698,7 @@ def test_asset_create_embargo(
     ).json()
     new_asset = Asset.objects.get(asset_id=resp['asset_id'])
 
+    assert new_asset.full_metadata['access'][0]['status'] == AccessType.EmbargoedAccess.value
     assert new_asset.blob.embargoed
     assert new_asset.zarr is None