feat(Metadata): add generic metadata and link with dataset, version, files

config/graphql/schema.graphql

@@ -5,5 +5,6 @@ scalar JSON
 scalar UUID
 scalar Generic
 scalar BigInt
+scalar OpaqueID
 type Query
 type Mutation

config/schema.py

@@ -1,3 +1,4 @@
+import base64
 import pathlib
 from uuid import UUID
 
@@ -7,12 +8,14 @@
     make_executable_schema,
     snake_case_fallback_resolvers,
 )
+from django.contrib.contenttypes.models import ContentType
 
 from hexa.core.schema import config_bindables, config_type_defs
 from hexa.countries.schema import countries_bindables, countries_type_defs
 from hexa.databases.schema import databases_bindables, databases_types_def
 from hexa.datasets.schema import datasets_bindables, datasets_type_defs
 from hexa.files.schema import files_bindables, files_type_def
+from hexa.metadata.schema import metadata_bindables, metadata_type_def
 from hexa.notebooks.schema import notebooks_bindables, notebooks_type_defs
 from hexa.pipelines.schema import pipelines_bindables, pipelines_type_defs
 from hexa.plugins.connector_accessmod.schema import (
@@ -31,6 +34,7 @@
 from hexa.workspaces.schema import workspaces_bindables, workspaces_type_def
 
 uuid_scalar = ScalarType("UUID")
+opaque_id_scalar = ScalarType("OpaqueID")
 
 
 @uuid_scalar.value_parser
@@ -42,6 +46,29 @@ def parse_uuid_value(value):
         raise ValueError(f'"{value}" is not a valid uuid')
 
 
+@opaque_id_scalar.value_parser
+def parse_opaque_id_value(value):
+    """Decodes base64 value and returns its object instance
+
+    Raises
+    ------
+        ObjectDoesNotExist: If the object instance or the content type does not exist
+    """
+    base64_decoded_id = base64.b64decode(value).decode("utf-8")
+    instance_id, content_type_id = base64_decoded_id.split(":")
+    content_type = ContentType.objects.get_for_id(content_type_id)
+    model_instance = content_type.model_class().objects.get(id=instance_id)
+    return model_instance
+
+
+@opaque_id_scalar.serializer
+def serialize_opaque_id(value):
+    """Encodes object instance id and content type to base64"""
+    content_type = ContentType.objects.get_for_model(value)
+    value = f"{value.id}:{content_type.id}"
+    return base64.b64encode(value.encode("utf-8")).decode("utf-8")
+
+
 type_defs = load_schema_from_path(
     f"{pathlib.Path(__file__).parent.resolve()}/graphql/schema.graphql"
 )
@@ -58,13 +85,15 @@ def parse_uuid_value(value):
         notebooks_type_defs,
         pipelines_type_defs,
         workspaces_type_def,
+        metadata_type_def,
         databases_types_def,
         files_type_def,
         config_type_defs,
         datasets_type_defs,
     ],
     [
         uuid_scalar,
+        opaque_id_scalar,
         *pipelines_bindables,
         *identity_bindables,
         *tags_bindables,
@@ -75,6 +104,7 @@ def parse_uuid_value(value):
         *countries_bindables,
         *notebooks_bindables,
         *workspaces_bindables,
+        *metadata_bindables,
         *databases_bindables,
         *files_bindables,
         *datasets_bindables,

config/settings/base.py

@@ -155,6 +155,7 @@
     "hexa.plugins.connector_accessmod",
     "hexa.workspaces",
     "hexa.databases",
+    "hexa.metadata",
     "hexa.files",
     "hexa.datasets",
     "django_otp",

docker-compose.yaml

@@ -54,7 +54,7 @@ services:
       - db
 
   # This service is only used for the connector_accessmod app.
-  dataworker:
+  accessmod_dataworker:
     <<: *common
     command: "manage validate_fileset_worker"
     restart: unless-stopped

hexa/datasets/graphql/schema.graphql

@@ -2,7 +2,7 @@
 """
 Dataset is a collection of files that are related to each other and are versioned.
 """
-type Dataset {
+type Dataset implements MetadataObject{
   id: ID!
   slug: String!
   name: String!
@@ -16,6 +16,8 @@ type Dataset {
   version(id: ID!): DatasetVersion
   latestVersion: DatasetVersion
   links(page: Int = 1, perPage: Int = 15): DatasetLinkPage!
+  attributes: [MetadataAttribute!]!
+  targetId: OpaqueID!
 }
 
 """
@@ -34,7 +36,7 @@ type DatasetPermissions {
 A version of a dataset. A version is a snapshot of the dataset at a point in time.
 
 """
-type DatasetVersion {
+type DatasetVersion implements MetadataObject{
   id: ID!
   name: String!
   description: String
@@ -44,6 +46,8 @@ type DatasetVersion {
   permissions: DatasetVersionPermissions!
   fileByName(name: String!): DatasetVersionFile
   files(page: Int = 1, perPage: Int = 15): DatasetVersionFilePage!
+  attributes: [MetadataAttribute!]!
+  targetId: OpaqueID!
 }
 
 """
@@ -92,33 +96,37 @@ type DatasetVersionPermissions {
 }
 
 """
-Statuses that can occur when generating file metadata
+Statuses that can occur when generating file sample
 """
-enum FileMetadataStatus{
+enum FileSampleStatus{
     PROCESSING,
     FAILED,
     FINISHED
 }
 
 """
-Metadata for dataset file
+File sample for dataset file
 """
-type DatasetFileMetadata {
-    sample: JSON!
-    status: FileMetadataStatus!
+type DatasetFileSample {
+    sample: JSON
+    status: FileSampleStatus!
+    statusReason: String
 }
 
 """
 A file in a dataset version.
 """
-type DatasetVersionFile {
+type DatasetVersionFile implements MetadataObject{
   id: ID!
   uri: String!
   filename: String!
   createdAt: DateTime!
   createdBy: User
   contentType: String!
-  fileMetadata: DatasetFileMetadata
+  fileSample: DatasetFileSample
+  properties: JSON
+  attributes: [MetadataAttribute!]!
+  targetId: OpaqueID!
 }
 
 """
@@ -161,7 +169,6 @@ extend type WorkspacePermissions {
   createDataset: Boolean!
 }
 
-
 """
 Input for creating a dataset.
 """

hexa/datasets/migrations/0009_datasetfilesample_delete_datasetfilemetadata.py

@@ -0,0 +1,59 @@
+# Generated by Django 4.2.16 on 2024-09-20 11:33
+
+import uuid
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("datasets", "0008_alter_datasetfilemetadata_options"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="DatasetFileSample",
+            fields=[
+                (
+                    "id",
+                    models.UUIDField(
+                        default=uuid.uuid4,
+                        editable=False,
+                        primary_key=True,
+                        serialize=False,
+                    ),
+                ),
+                ("created_at", models.DateTimeField(auto_now_add=True)),
+                ("updated_at", models.DateTimeField(auto_now=True)),
+                ("sample", models.JSONField(blank=True, default=list, null=True)),
+                (
+                    "status",
+                    models.CharField(
+                        choices=[
+                            ("PROCESSING", "Processing"),
+                            ("FAILED", "Failed"),
+                            ("FINISHED", "Finished"),
+                        ],
+                        default="PROCESSING",
+                        max_length=10,
+                    ),
+                ),
+                ("status_reason", models.TextField(blank=True, null=True)),
+                (
+                    "dataset_version_file",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="samples",
+                        to="datasets.datasetversionfile",
+                    ),
+                ),
+            ],
+            options={
+                "ordering": ["-created_at"],
+            },
+        ),
+        migrations.DeleteModel(
+            name="DatasetFileMetadata",
+        ),
+    ]

hexa/datasets/migrations/0010_datasetversionfile_properties.py

@@ -0,0 +1,17 @@
+# Generated by Django 4.2.16 on 2024-10-09 07:45
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("datasets", "0009_datasetfilesample_delete_datasetfilemetadata"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="datasetversionfile",
+            name="properties",
+            field=models.JSONField(default=dict),
+        ),
+    ]