Folder And Dataset Metadata

CHANGELOG.unreleased.md

@@ -13,6 +13,7 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released
 ### Added
 - WEBKNOSSOS now automatically searches in subfolder / sub-collection identifiers for valid datasets in case a provided link to a remote dataset does not directly point to a dataset. [#7912](https://github.com/scalableminds/webknossos/pull/7912)
 - Added the option to move a bounding box via dragging while pressing ctrl / meta. [#7892](https://github.com/scalableminds/webknossos/pull/7892)
+- Added the option to add metadata entries to datasets and folders. The metadata can be viewed and edited in the dashboard in the right details tab.[#7886](https://github.com/scalableminds/webknossos/pull/7886)
 - Added route `/import?url=<url_to_datasource>` to automatically import and view remote datasets. [#7844](https://github.com/scalableminds/webknossos/pull/7844)
 - Added that newly created, modified and clicked on bounding boxes are now highlighted and scrolled into view, while the bounding box tool is active. [#7935](https://github.com/scalableminds/webknossos/pull/7935)
 - The configured unit in the dataset upload view is now passed to the convert_to_wkw worker job. [#7970](https://github.com/scalableminds/webknossos/pull/7970)

MIGRATIONS.unreleased.md

@@ -11,3 +11,4 @@ User-facing changes are documented in the [changelog](CHANGELOG.released.md).
 - If Segment Anything was already configured, it needs to be pointed to an endpoint that works with SAM 2. [#7965](https://github.com/scalableminds/webknossos/pull/7965)
 
 ### Postgres Evolutions:
+- [119-add-metadata-to-folders-and-datasets.sql](conf/evolutions/119-add-metadata-to-folders-and-datasets.sql)

app/controllers/DatasetController.scala

@@ -36,6 +36,7 @@ case class DatasetUpdateParameters(
     sortingKey: Option[Instant],
     isPublic: Option[Boolean],
     tags: Option[List[String]],
+    metadata: Option[JsArray],
     folderId: Option[ObjectId]
 )
 
@@ -94,6 +95,7 @@ class DatasetController @Inject()(userService: UserService,
       (__ \ "sortingKey").readNullable[Instant] and
       (__ \ "isPublic").read[Boolean] and
       (__ \ "tags").read[List[String]] and
+      (__ \ "metadata").readNullable[JsArray] and
       (__ \ "folderId").readNullable[ObjectId]).tupled
 
   def removeFromThumbnailCache(organizationName: String, datasetName: String): Action[AnyContent] =
@@ -313,18 +315,22 @@ class DatasetController @Inject()(userService: UserService,
   def update(organizationName: String, datasetName: String): Action[JsValue] =
     sil.SecuredAction.async(parse.json) { implicit request =>
       withJsonBodyUsing(datasetPublicReads) {
-        case (description, displayName, sortingKey, isPublic, tags, folderId) =>
+        case (description, displayName, sortingKey, isPublic, tags, metadata, folderId) =>
           for {
             dataset <- datasetDAO.findOneByNameAndOrganization(datasetName, request.identity._organization) ?~> notFoundMessage(
               datasetName) ~> NOT_FOUND
+            maybeUpdatedMetadata = metadata.getOrElse(dataset.metadata)
             _ <- Fox.assertTrue(datasetService.isEditableBy(dataset, Some(request.identity))) ?~> "notAllowed" ~> FORBIDDEN
-            _ <- datasetDAO.updateFields(dataset._id,
-                                         description,
-                                         displayName,
-                                         sortingKey.getOrElse(dataset.created),
-                                         isPublic,
-                                         folderId.getOrElse(dataset._folder))
-            _ <- datasetDAO.updateTags(dataset._id, tags)
+            _ <- datasetDAO.updateFields(
+              dataset._id,
+              description,
+              displayName,
+              sortingKey.getOrElse(dataset.created),
+              isPublic,
+              tags,
+              maybeUpdatedMetadata,
+              folderId.getOrElse(dataset._folder)
+            )
             updated <- datasetDAO.findOneByNameAndOrganization(datasetName, request.identity._organization)
             _ = analyticsService.track(ChangeDatasetSettingsEvent(request.identity, updated))
             js <- datasetService.publicWrites(updated, Some(request.identity))

app/controllers/FolderController.scala

@@ -7,7 +7,7 @@ import models.folder.{Folder, FolderDAO, FolderParameters, FolderService}
 import models.organization.OrganizationDAO
 import models.team.{TeamDAO, TeamService}
 import models.user.UserService
-import play.api.libs.json.Json
+import play.api.libs.json.{JsArray, Json}
 import play.api.mvc.{Action, AnyContent, PlayBodyParsers}
 import security.WkEnv
 import utils.ObjectId
@@ -53,6 +53,7 @@ class FolderController @Inject()(
         _ <- folderDAO.findOne(idValidated) ?~> "folder.notFound"
         - <- Fox.assertTrue(folderDAO.isEditable(idValidated)) ?~> "folder.update.notAllowed" ~> FORBIDDEN
         _ <- folderService.assertValidFolderName(params.name)
+        _ <- folderDAO.updateMetadata(idValidated, params.metadata)
         _ <- folderDAO.updateName(idValidated, params.name) ?~> "folder.update.name.failed"
         _ <- folderService
           .updateAllowedTeams(idValidated, params.allowedTeams, request.identity) ?~> "folder.update.teams.failed"
@@ -103,7 +104,7 @@ class FolderController @Inject()(
     for {
       parentIdValidated <- ObjectId.fromString(parentId)
       _ <- folderService.assertValidFolderName(name)
-      newFolder = Folder(ObjectId.generate, name)
+      newFolder = Folder(ObjectId.generate, name, JsArray.empty)
       _ <- folderDAO.findOne(parentIdValidated) ?~> "folder.notFound"
       _ <- folderDAO.insertAsChild(parentIdValidated, newFolder) ?~> "folder.create.failed"
       organization <- organizationDAO.findOne(request.identity._organization) ?~> "folder.notFound"

app/controllers/InitialDataController.scala

@@ -13,7 +13,7 @@ import models.task.{TaskType, TaskTypeDAO}
 import models.team._
 import models.user._
 import net.liftweb.common.{Box, Full}
-import play.api.libs.json.Json
+import play.api.libs.json.{JsArray, Json}
 import utils.{ObjectId, StoreModules, WkConf}
 
 import javax.inject.Inject
@@ -169,7 +169,8 @@ Samplecountry
   private def insertRootFolder(): Fox[Unit] =
     folderDAO.findOne(defaultOrganization._rootFolder).futureBox.flatMap {
       case Full(_) => Fox.successful(())
-      case _       => folderDAO.insertAsRoot(Folder(defaultOrganization._rootFolder, folderService.defaultRootName))
+      case _ =>
+        folderDAO.insertAsRoot(Folder(defaultOrganization._rootFolder, folderService.defaultRootName, JsArray.empty))
     }
 
   private def insertDefaultUser(userEmail: String,

app/models/dataset/Dataset.scala

@@ -55,7 +55,7 @@ case class Dataset(_id: ObjectId,
                    status: String,
                    logoUrl: Option[String],
                    sortingKey: Instant = Instant.now,
-                   details: Option[JsObject] = None,
+                   metadata: JsArray = JsArray.empty,
                    tags: List[String] = List.empty,
                    created: Instant = Instant.now,
                    isDeleted: Boolean = false)
@@ -119,7 +119,7 @@ class DatasetDAO @Inject()(sqlClient: SqlClient, datasetLayerDAO: DatasetLayerDA
         JsonHelper.parseAndValidateJson[DatasetViewConfiguration](_))
       adminViewConfigurationOpt <- Fox.runOptional(r.adminviewconfiguration)(
         JsonHelper.parseAndValidateJson[DatasetViewConfiguration](_))
-      details <- Fox.runOptional(r.details)(JsonHelper.parseAndValidateJson[JsObject](_))
+      metadata <- JsonHelper.parseAndValidateJson[JsArray](r.metadata)
     } yield {
       Dataset(
         ObjectId(r._Id),
@@ -141,7 +141,7 @@ class DatasetDAO @Inject()(sqlClient: SqlClient, datasetLayerDAO: DatasetLayerDA
         r.status,
         r.logourl,
         Instant.fromSql(r.sortingkey),
-        details,
+        metadata,
         parseArrayLiteral(r.tags).sorted,
         Instant.fromSql(r.created),
         r.isdeleted
@@ -493,6 +493,7 @@ class DatasetDAO @Inject()(sqlClient: SqlClient, datasetLayerDAO: DatasetLayerDA
       params.isPublic.map(v => q"isPublic = $v"),
       params.tags.map(v => q"tags = $v"),
       params.folderId.map(v => q"_folder = $v"),
+      params.metadata.map(v => q"metadata = $v"),
     ).flatten
     if (setQueries.isEmpty) {
       Fox.successful(())
@@ -509,18 +510,24 @@ class DatasetDAO @Inject()(sqlClient: SqlClient, datasetLayerDAO: DatasetLayerDA
     }
   }
 
-  def updateFields(_id: ObjectId,
+  def updateFields(datasetId: ObjectId,
                    description: Option[String],
                    displayName: Option[String],
                    sortingKey: Instant,
                    isPublic: Boolean,
+                   tags: List[String],
+                   metadata: JsArray,
                    folderId: ObjectId)(implicit ctx: DBAccessContext): Fox[Unit] = {
-    val query = for { row <- Datasets if notdel(row) && row._Id === _id.id } yield
-      (row.description, row.displayname, row.sortingkey, row.ispublic, row._Folder)
-    for {
-      _ <- assertUpdateAccess(_id)
-      _ <- run(query.update(description, displayName, sortingKey.toSql, isPublic, folderId.toString))
-    } yield ()
+    val updateParameters = new DatasetUpdateParameters(
+      description = Some(description),
+      displayName = Some(displayName),
+      sortingKey = Some(sortingKey),
+      isPublic = Some(isPublic),
+      tags = Some(tags),
+      metadata = Some(metadata),
+      folderId = Some(folderId)
+    )
+    updatePartial(datasetId, updateParameters)
   }
 
   def updateTags(id: ObjectId, tags: List[String])(implicit ctx: DBAccessContext): Fox[Unit] =
@@ -564,7 +571,7 @@ class DatasetDAO @Inject()(sqlClient: SqlClient, datasetLayerDAO: DatasetLayerDA
                      inboxSourceHash, defaultViewConfiguration, adminViewConfiguration,
                      description, displayName, isPublic, isUsable,
                      name, voxelSizeFactor, voxelSizeUnit, status,
-                     sharingToken, sortingKey, details, tags,
+                     sharingToken, sortingKey, metadata, tags,
                      created, isDeleted
                    )
                    VALUES(
@@ -573,7 +580,7 @@ class DatasetDAO @Inject()(sqlClient: SqlClient, datasetLayerDAO: DatasetLayerDA
                      ${d.inboxSourceHash}, $defaultViewConfiguration, $adminViewConfiguration,
                      ${d.description}, ${d.displayName}, ${d.isPublic}, ${d.isUsable},
                      ${d.name}, ${d.voxelSize.map(_.factor)}, ${d.voxelSize.map(_.unit)}, ${d.status.take(1024)},
-                     ${d.sharingToken}, ${d.sortingKey}, ${d.details}, ${d.tags},
+                     ${d.sharingToken}, ${d.sortingKey}, ${d.metadata}, ${d.tags},
                      ${d.created}, ${d.isDeleted}
                    )""".asUpdate)
     } yield ()

app/models/dataset/DatasetService.scala

@@ -90,8 +90,15 @@ class DatasetService @Inject()(organizationDAO: OrganizationDAO,
   ): Fox[Dataset] = {
     implicit val ctx: DBAccessContext = GlobalAccessContext
     val newId = ObjectId.generate
-    val details =
-      Json.obj("species" -> "species name", "brainRegion" -> "brain region", "acquisition" -> "acquisition method")
+    val metadata =
+      if (publication.isDefined)
+        Json.arr(
+          Json.obj("type" -> "string", "key" -> "species", "value" -> "species name"),
+          Json.obj("type" -> "string", "key" -> "brainRegion", "value" -> "brain region"),
+          Json.obj("type" -> "string", "key" -> "acquisition", "value" -> "acquisition method")
+        )
+      else Json.arr()
+
     val dataSourceHash = if (dataSource.isUsable) Some(dataSource.hashCode()) else None
     for {
       organization <- organizationDAO.findOneByName(owningOrganization)
@@ -115,7 +122,7 @@ class DatasetService @Inject()(organizationDAO: OrganizationDAO,
         sharingToken = None,
         status = dataSource.statusOpt.getOrElse(""),
         logoUrl = None,
-        details = publication.map(_ => details)
+        metadata = metadata
       )
       _ <- datasetDAO.insertOne(dataset)
       _ <- datasetDataLayerDAO.updateLayers(newId, dataSource)
@@ -366,7 +373,7 @@ class DatasetService @Inject()(organizationDAO: OrganizationDAO,
         "lastUsedByUser" -> lastUsedByUser,
         "logoUrl" -> logoUrl,
         "sortingKey" -> dataset.sortingKey,
-        "details" -> dataset.details,
+        "metadata" -> dataset.metadata,
         "isUnreported" -> Json.toJson(isUnreported(dataset)),
         "tags" -> dataset.tags,
         "folderId" -> dataset._folder,

app/models/folder/Folder.scala

@@ -1,14 +1,14 @@
 package models.folder
 
 import com.scalableminds.util.accesscontext.DBAccessContext
-import com.scalableminds.util.tools.Fox
+import com.scalableminds.util.tools.{Fox, JsonHelper}
 import com.scalableminds.util.tools.Fox.{bool2Fox, option2Fox}
 import com.scalableminds.webknossos.schema.Tables._
 import com.typesafe.scalalogging.LazyLogging
 import models.organization.{Organization, OrganizationDAO}
 import models.team.{TeamDAO, TeamService}
 import models.user.User
-import play.api.libs.json.{JsObject, Json, OFormat}
+import play.api.libs.json.{JsArray, JsObject, Json, OFormat}
 import slick.jdbc.PostgresProfile.api._
 import slick.lifted.Rep
 import slick.sql.SqlAction
@@ -19,11 +19,11 @@ import javax.inject.Inject
 import scala.annotation.tailrec
 import scala.concurrent.ExecutionContext
 
-case class Folder(_id: ObjectId, name: String)
+case class Folder(_id: ObjectId, name: String, metadata: JsArray)
 
-case class FolderWithParent(_id: ObjectId, name: String, _parent: Option[ObjectId])
+case class FolderWithParent(_id: ObjectId, name: String, metadata: JsArray, _parent: Option[ObjectId])
 
-case class FolderParameters(name: String, allowedTeams: List[ObjectId])
+case class FolderParameters(name: String, allowedTeams: List[ObjectId], metadata: JsArray)
 object FolderParameters {
   implicit val jsonFormat: OFormat[FolderParameters] = Json.format[FolderParameters]
 }
@@ -48,17 +48,21 @@ class FolderService @Inject()(teamDAO: TeamDAO,
         teamService.publicWrites(t, requestingUserOrganization)) ?~> "dataset.list.teamWritesFailed"
       isEditable <- folderDAO.isEditable(folder._id)
     } yield
-      Json.obj("id" -> folder._id,
-               "name" -> folder.name,
-               "allowedTeams" -> teamsJs,
-               "allowedTeamsCumulative" -> teamsCumulativeJs,
-               "isEditable" -> isEditable)
+      Json.obj(
+        "id" -> folder._id,
+        "name" -> folder.name,
+        "metadata" -> folder.metadata,
+        "allowedTeams" -> teamsJs,
+        "allowedTeamsCumulative" -> teamsCumulativeJs,
+        "isEditable" -> isEditable
+      )
 
   def publicWritesWithParent(folderWithParent: FolderWithParent, allEditableIds: Set[ObjectId]): JsObject =
     Json.obj(
       "id" -> folderWithParent._id,
       "name" -> folderWithParent.name,
       "parent" -> folderWithParent._parent,
+      "metadata" -> folderWithParent.metadata,
       "isEditable" -> allEditableIds.contains(folderWithParent._id)
     )
 
@@ -117,7 +121,7 @@ class FolderService @Inject()(teamDAO: TeamDAO,
     remainingPathNames match {
       case pathNamesHead :: pathNamesTail =>
         for {
-          newFolder <- Fox.successful(Folder(ObjectId.generate, pathNamesHead))
+          newFolder <- Fox.successful(Folder(ObjectId.generate, pathNamesHead, JsArray.empty))
           _ <- folderDAO.insertAsChild(parentFolderId, newFolder)
           folderId <- createMissingFoldersForPathNames(newFolder._id, pathNamesTail)
         } yield folderId
@@ -133,10 +137,18 @@ class FolderDAO @Inject()(sqlClient: SqlClient)(implicit ec: ExecutionContext)
   protected def isDeletedColumn(x: Folders): Rep[Boolean] = x.isdeleted
 
   protected def parse(r: FoldersRow): Fox[Folder] =
-    Fox.successful(Folder(ObjectId(r._Id), r.name))
+    for {
+      metadata <- parseMetadata(r.metadata)
+    } yield Folder(ObjectId(r._Id), r.name, metadata)
+
+  private def parseWithParent(t: (String, String, String, Option[String])): Fox[FolderWithParent] =
+    for {
+      metadata <- parseMetadata(t._3)
+      folderWithParent = FolderWithParent(ObjectId(t._1), t._2, metadata, t._4.map(ObjectId(_)))
+    } yield folderWithParent
 
-  private def parseWithParent(t: (String, String, Option[String])): Fox[FolderWithParent] =
-    Fox.successful(FolderWithParent(ObjectId(t._1), t._2, t._3.map(ObjectId(_))))
+  private def parseMetadata(literal: String): Fox[JsArray] =
+    JsonHelper.parseAndValidateJson[JsArray](literal)
 
   override protected def readAccessQ(requestingUserId: ObjectId): SqlToken =
     readAccessQWithPrefix(requestingUserId, q"")
@@ -246,6 +258,12 @@ class FolderDAO @Inject()(sqlClient: SqlClient)(implicit ec: ExecutionContext)
       _ <- run(q"UPDATE webknossos.folders SET name = $name WHERE _id = $folderId".asUpdate)
     } yield ()
 
+  def updateMetadata(folderId: ObjectId, metadata: JsArray)(implicit ctx: DBAccessContext): Fox[Unit] =
+    for {
+      _ <- assertUpdateAccess(folderId)
+      _ <- run(q"UPDATE webknossos.folders SET metadata = $metadata WHERE _id = $folderId".asUpdate)
+    } yield ()
+
   def findAllEditableIds(implicit ctx: DBAccessContext): Fox[List[ObjectId]] =
     for {
       updateAccessQuery <- accessQueryFromAccessQ(updateAccessQ)
@@ -269,7 +287,7 @@ class FolderDAO @Inject()(sqlClient: SqlClient)(implicit ec: ExecutionContext)
     for {
       accessQueryWithPrefix <- accessQueryFromAccessQWithPrefix(readAccessQWithPrefix, prefix = q"f.")
       accessQuery <- readAccessQuery
-      rows <- run(q"""SELECT f._id, f.name, fp._ancestor
+      rows <- run(q"""SELECT f._id, f.name, f.metadata, fp._ancestor
               FROM webknossos.folders_ f
               JOIN webknossos.folder_paths fp -- join to find immediate parent, this will also kick out self
               ON f._id = fp._descendant
@@ -278,11 +296,11 @@ class FolderDAO @Inject()(sqlClient: SqlClient)(implicit ec: ExecutionContext)
               FROM webknossos.folder_paths
               WHERE _ancestor = $folderId)
               AND $accessQueryWithPrefix
-              UNION ALL SELECT _id, name, NULL -- find self again, with no parent
+              UNION ALL SELECT _id, name, metadata, NULL -- find self again, with no parent
               FROM webknossos.folders_
               WHERE _id = $folderId
               AND $accessQuery
-              """.as[(String, String, Option[String])])
+              """.as[(String, String, String, Option[String])])
       parsed <- Fox.combined(rows.toList.map(parseWithParent))
     } yield parsed
 

app/models/organization/OrganizationService.scala

@@ -10,7 +10,7 @@ import models.dataset.{DataStore, DataStoreDAO}
 import models.folder.{Folder, FolderDAO, FolderService}
 import models.team.{PricingPlan, Team, TeamDAO}
 import models.user.{Invite, MultiUserDAO, User, UserDAO, UserService}
-import play.api.libs.json.{JsObject, Json}
+import play.api.libs.json.{JsArray, JsObject, Json}
 import utils.{ObjectId, WkConf}
 
 import scala.concurrent.{ExecutionContext, Future}
@@ -111,7 +111,7 @@ class OrganizationService @Inject()(organizationDAO: OrganizationDAO,
       _ <- bool2Fox(existingOrganization.isEmpty) ?~> "organization.name.alreadyInUse"
       initialPricingParameters = if (conf.Features.isWkorgInstance) (PricingPlan.Basic, Some(3), Some(50000000000L))
       else (PricingPlan.Custom, None, None)
-      organizationRootFolder = Folder(ObjectId.generate, folderService.defaultRootName)
+      organizationRootFolder = Folder(ObjectId.generate, folderService.defaultRootName, JsArray.empty)
 
       organization = Organization(
         ObjectId.generate,