Merge branch 'master' of github.com:scalableminds/webknossos into all…

…ow-perma-dataset-rotation

CHANGELOG.released.md

@@ -451,7 +451,7 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released
 ### Fixed
 - Fixed that some segment (group) actions were not properly disabled for non-editable segmentation layers. [#7207](https://github.com/scalableminds/webknossos/issues/7207)
 - Fixed a bug where data from zarr2 datasets that have a channel axis was broken. [#7374](https://github.com/scalableminds/webknossos/pull/7374)
-- Fixed a bug which changed the cursor position while editing the name of a tree or the comment of a node. [#7390](#https://github.com/scalableminds/webknossos/pull/7390)
+- Fixed a bug which changed the cursor position while editing the name of a tree or the comment of a node. [#7390](https://github.com/scalableminds/webknossos/pull/7390)
 - Streaming sharded zarr3 datasets from servers which do not respond with Accept-Ranges header is now possible. [#7392](https://github.com/scalableminds/webknossos/pull/7392)
 
 ## [23.10.2](https://github.com/scalableminds/webknossos/releases/tag/23.10.2) - 2023-09-26

CHANGELOG.unreleased.md

@@ -32,10 +32,12 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released
 - Fixed a bug where changing the color of a segment via the menu in the segments tab would update the segment color of the previous segment, on which the context menu was opened. [#8225](https://github.com/scalableminds/webknossos/pull/8225)
 - Fixed a bug where in the add remote dataset view the dataset name setting was not in sync with the datasource setting of the advanced tab making the form not submittable. [#8245](https://github.com/scalableminds/webknossos/pull/8245)
 - Fixed a bug when importing an NML with groups when only groups but no trees exist in an annotation. [#8176](https://github.com/scalableminds/webknossos/pull/8176)
+- Fix read and update dataset route for versions 8 and lower. [#8263](https://github.com/scalableminds/webknossos/pull/8263)
 - Added missing legacy support for `isValidNewName` route. [#8252](https://github.com/scalableminds/webknossos/pull/8252)
 - Fixed a bug where trying to delete a non-existing node (via the API, for example) would delete the whole active tree. [#8176](https://github.com/scalableminds/webknossos/pull/8176)
 - Fixed a bug where dataset uploads would fail if the organization directory on disk is missing. [#8230](https://github.com/scalableminds/webknossos/pull/8230)
 - Fixed some layout issues in the upload view. [#8231](https://github.com/scalableminds/webknossos/pull/8231)
+- Fixed `FATAL: role "postgres" does not exist` error message in Docker compose. [#8240](https://github.com/scalableminds/webknossos/pull/8240)
 
 ### Removed
 - Removed support for HTTP API versions 3 and 4. [#8075](https://github.com/scalableminds/webknossos/pull/8075)

app/controllers/DatasetController.scala

@@ -176,91 +176,70 @@ class DatasetController @Inject()(userService: UserService,
       // Change output format to return only a compact list with essential information on the datasets
       compact: Option[Boolean]
   ): Action[AnyContent] = sil.UserAwareAction.async { implicit request =>
-    log() {
-      for {
-        folderIdValidated <- Fox.runOptional(folderId)(ObjectId.fromString)
-        uploaderIdValidated <- Fox.runOptional(uploaderId)(ObjectId.fromString)
-        organizationIdOpt = if (onlyMyOrganization.getOrElse(false))
-          request.identity.map(_._organization)
-        else
-          organizationId
-        js <- if (compact.getOrElse(false)) {
-          for {
-            datasetInfos <- datasetDAO.findAllCompactWithSearch(
-              isActive,
-              isUnreported,
-              organizationIdOpt,
-              folderIdValidated,
-              uploaderIdValidated,
-              searchQuery,
-              request.identity.map(_._id),
-              recursive.getOrElse(false),
-              limitOpt = limit
-            )
-          } yield Json.toJson(datasetInfos)
-        } else {
-          for {
-            _ <- Fox.successful(())
-            _ = logger.info(
-              s"Requesting listing datasets with isActive '$isActive', isUnreported '$isUnreported', organizationId '$organizationIdOpt', folderId '$folderIdValidated', uploaderId '$uploaderIdValidated', searchQuery '$searchQuery', recursive '$recursive', limit '$limit'")
-            datasets <- datasetDAO.findAllWithSearch(isActive,
-                                                     isUnreported,
-                                                     organizationIdOpt,
-                                                     folderIdValidated,
-                                                     uploaderIdValidated,
-                                                     searchQuery,
-                                                     recursive.getOrElse(false),
-                                                     limit) ?~> "dataset.list.failed" ?~> "Dataset listing failed"
-            _ = logger.info(s"Found ${datasets.size} datasets successfully")
-            js <- listGrouped(datasets, request.identity) ?~> "dataset.list.failed" ?~> "Grouping datasets failed"
-          } yield Json.toJson(js)
-        }
-        _ = Fox.runOptional(request.identity)(user => userDAO.updateLastActivity(user._id))
-      } yield addRemoteOriginHeaders(Ok(js))
-    }
+    for {
+      folderIdValidated <- Fox.runOptional(folderId)(ObjectId.fromString)
+      uploaderIdValidated <- Fox.runOptional(uploaderId)(ObjectId.fromString)
+      organizationIdOpt = if (onlyMyOrganization.getOrElse(false))
+        request.identity.map(_._organization)
+      else
+        organizationId
+      js <- if (compact.getOrElse(false)) {
+        for {
+          datasetInfos <- datasetDAO.findAllCompactWithSearch(
+            isActive,
+            isUnreported,
+            organizationIdOpt,
+            folderIdValidated,
+            uploaderIdValidated,
+            searchQuery,
+            request.identity.map(_._id),
+            recursive.getOrElse(false),
+            limitOpt = limit
+          )
+        } yield Json.toJson(datasetInfos)
+      } else {
+        for {
+          datasets <- datasetDAO.findAllWithSearch(isActive,
+                                                   isUnreported,
+                                                   organizationIdOpt,
+                                                   folderIdValidated,
+                                                   uploaderIdValidated,
+                                                   searchQuery,
+                                                   recursive.getOrElse(false),
+                                                   limit) ?~> "dataset.list.failed"
+          js <- listGrouped(datasets, request.identity) ?~> "dataset.list.grouping.failed"
+        } yield Json.toJson(js)
+      }
+      _ = Fox.runOptional(request.identity)(user => userDAO.updateLastActivity(user._id))
+    } yield addRemoteOriginHeaders(Ok(js))
   }
 
   private def listGrouped(datasets: List[Dataset], requestingUser: Option[User])(
       implicit ctx: DBAccessContext,
       m: MessagesProvider): Fox[List[JsObject]] =
     for {
-      _ <- Fox.successful(())
-      _ = logger.info(s"datasets: $datasets, requestingUser: ${requestingUser.map(_._id)}")
       requestingUserTeamManagerMemberships <- Fox.runOptional(requestingUser)(user =>
-        userService
-          .teamManagerMembershipsFor(user._id)) ?~> s"Could not find team manager memberships for user ${requestingUser
-        .map(_._id)}"
-      _ = logger.info(
-        s"requestingUserTeamManagerMemberships: ${requestingUserTeamManagerMemberships.map(_.map(_.toString))}")
+        userService.teamManagerMembershipsFor(user._id))
       groupedByOrga = datasets.groupBy(_._organization).toList
       js <- Fox.serialCombined(groupedByOrga) { byOrgaTuple: (String, List[Dataset]) =>
         for {
-          _ <- Fox.successful(())
-          _ = logger.info(s"byOrgaTuple orga: ${byOrgaTuple._1}, datasets: ${byOrgaTuple._2}")
-          organization <- organizationDAO.findOne(byOrgaTuple._1)(GlobalAccessContext) ?~> s"Could not find organization ${byOrgaTuple._1}"
+          organization <- organizationDAO.findOne(byOrgaTuple._1)(GlobalAccessContext) ?~> "organization.notFound"
           groupedByDataStore = byOrgaTuple._2.groupBy(_._dataStore).toList
-          _ <- Fox.serialCombined(groupedByDataStore) { byDataStoreTuple: (String, List[Dataset]) =>
-            {
-              logger.info(s"datastore: ${byDataStoreTuple._1}, datasets: ${byDataStoreTuple._2}")
-              Fox.successful(())
-            }
-          }
           result <- Fox.serialCombined(groupedByDataStore) { byDataStoreTuple: (String, List[Dataset]) =>
             for {
-              dataStore <- dataStoreDAO.findOneByName(byDataStoreTuple._1.trim)(GlobalAccessContext) ?~>
-                s"Could not find data store ${byDataStoreTuple._1}"
+              dataStore <- dataStoreDAO.findOneByName(byDataStoreTuple._1.trim)(GlobalAccessContext)
               resultByDataStore: Seq[JsObject] <- Fox.serialCombined(byDataStoreTuple._2) { d =>
                 datasetService.publicWrites(
                   d,
                   requestingUser,
                   Some(organization),
                   Some(dataStore),
                   requestingUserTeamManagerMemberships) ?~> Messages("dataset.list.writesFailed", d.name)
-              } ?~> "Could not find public writes for datasets"
+              }
             } yield resultByDataStore
-          } ?~> s"Could not group by datastore for datasets ${byOrgaTuple._2.map(_._id)}"
+          }
         } yield result.flatten
-      } ?~> s"Could not group by organization for datasets ${datasets.map(_._id)}"
+      }
     } yield js.flatten
 
   def accessList(datasetId: String): Action[AnyContent] = sil.SecuredAction.async { implicit request =>
@@ -339,7 +318,7 @@ class DatasetController @Inject()(userService: UserService,
     sil.SecuredAction.async(parse.json) { implicit request =>
       withJsonBodyUsing(datasetPublicReads) {
         case (description, datasetName, legacyDatasetDisplayName, sortingKey, isPublic, tags, metadata, folderId) => {
-          val name = if (datasetName.isDefined) datasetName else legacyDatasetDisplayName
+          val name = if (legacyDatasetDisplayName.isDefined) legacyDatasetDisplayName else datasetName
           for {
             datasetIdValidated <- ObjectId.fromString(datasetId)
             dataset <- datasetDAO.findOne(datasetIdValidated) ?~> notFoundMessage(datasetIdValidated.toString) ~> NOT_FOUND

app/controllers/LegacyApiController.scala

@@ -72,7 +72,8 @@ class LegacyApiController @Inject()(annotationController: AnnotationController,
       for {
         dataset <- datasetDAO.findOneByNameAndOrganization(datasetName, organizationId)
         result <- datasetController.read(dataset._id.toString, sharingToken)(request)
-      } yield result
+        adaptedResult <- replaceInResult(migrateDatasetJsonToOldFormat)(result)
+      } yield adaptedResult
     }
 
   def updateDatasetV8(organizationId: String, datasetName: String): Action[JsValue] =
@@ -81,7 +82,8 @@ class LegacyApiController @Inject()(annotationController: AnnotationController,
         _ <- Fox.successful(logVersioned(request))
         dataset <- datasetDAO.findOneByNameAndOrganization(datasetName, organizationId)
         result <- datasetController.update(dataset._id.toString)(request)
-      } yield result
+        adaptedResult <- replaceInResult(migrateDatasetJsonToOldFormat)(result)
+      } yield adaptedResult
     }
 
   def updateDatasetTeamsV8(organizationId: String, datasetName: String): Action[List[ObjectId]] =
@@ -241,6 +243,19 @@ class LegacyApiController @Inject()(annotationController: AnnotationController,
 
   /* private helper methods for legacy adaptation */
 
+  private def migrateDatasetJsonToOldFormat(jsResult: JsObject): Fox[JsObject] = {
+    val datasetName = (jsResult \ "name").asOpt[String]
+    val directoryName = (jsResult \ "directoryName").asOpt[String]
+    datasetName.zip(directoryName) match {
+      case Some((name, dirName)) =>
+        for {
+          dsWithOldNameField <- tryo(jsResult - "directoryName" + ("name" -> Json.toJson(dirName))).toFox
+          dsWithOldDisplayNameField <- tryo(dsWithOldNameField + ("displayName" -> Json.toJson(name))).toFox
+        } yield dsWithOldDisplayNameField
+      case _ => Fox.successful(jsResult)
+    }
+  }
+
   private def addDataSetToTaskInAnnotation(jsResult: JsObject): Fox[JsObject] = {
     val taskObjectOpt = (jsResult \ "task").asOpt[JsObject]
     taskObjectOpt

app/models/annotation/AnnotationService.scala

@@ -179,7 +179,8 @@ class AnnotationService @Inject()(
         mappingName = mappingName,
         mags = magsRestricted.map(vec3IntToProto),
         hasSegmentIndex = Some(fallbackLayer.isEmpty || fallbackLayerHasSegmentIndex),
-        additionalAxes = AdditionalAxis.toProto(additionalAxes)
+        additionalAxes = AdditionalAxis.toProto(additionalAxes),
+        volumeBucketDataHasChanged = Some(false)
       )
   }
 

app/models/annotation/nml/NmlParser.scala

@@ -83,7 +83,7 @@ class NmlParser @Inject()(datasetDAO: DatasetDAO) extends LazyLogging with Proto
             zoomLevel = nmlParams.zoomLevel,
             userBoundingBox = None,
             userBoundingBoxes = nmlParams.userBoundingBoxes,
-            organizationId = Some(nmlParams.organizationId),
+            organizationId = Some(dataset._organization),
             segments = v.segments,
             mappingName = v.mappingName,
             mappingIsLocked = v.mappingIsLocked,
@@ -113,7 +113,7 @@ class NmlParser @Inject()(datasetDAO: DatasetDAO) extends LazyLogging with Proto
             None,
             nmlParams.treeGroupsAfterSplit,
             nmlParams.userBoundingBoxes,
-            Some(nmlParams.organizationId),
+            Some(dataset._organization),
             nmlParams.editPositionAdditionalCoordinates,
             additionalAxes = nmlParams.additionalAxisProtos
           )

app/models/dataset/Dataset.scala

@@ -115,14 +115,12 @@ class DatasetDAO @Inject()(sqlClient: SqlClient, datasetLayerDAO: DatasetLayerDA
 
   protected def parse(r: DatasetsRow): Fox[Dataset] =
     for {
-      voxelSize <- parseVoxelSizeOpt(r.voxelsizefactor, r.voxelsizeunit) ?~> "could not parse dataset voxel size"
+      voxelSize <- parseVoxelSizeOpt(r.voxelsizefactor, r.voxelsizeunit)
       defaultViewConfigurationOpt <- Fox.runOptional(r.defaultviewconfiguration)(
-        JsonHelper
-          .parseAndValidateJson[DatasetViewConfiguration](_)) ?~> "could not parse dataset default view configuration"
+        JsonHelper.parseAndValidateJson[DatasetViewConfiguration](_))
       adminViewConfigurationOpt <- Fox.runOptional(r.adminviewconfiguration)(
-        JsonHelper
-          .parseAndValidateJson[DatasetViewConfiguration](_)) ?~> "could not parse dataset admin view configuration"
-      metadata <- JsonHelper.parseAndValidateJson[JsArray](r.metadata) ?~> "could not parse dataset metadata"
+        JsonHelper.parseAndValidateJson[DatasetViewConfiguration](_))
+      metadata <- JsonHelper.parseAndValidateJson[JsArray](r.metadata)
     } yield {
       Dataset(
         ObjectId(r._Id),
@@ -220,11 +218,9 @@ class DatasetDAO @Inject()(sqlClient: SqlClient, datasetLayerDAO: DatasetLayerDA
                                                       includeSubfolders,
                                                       None,
                                                       None)
-      _ = logger.info(s"Requesting datasets with selection predicates '$selectionPredicates'")
       limitQuery = limitOpt.map(l => q"LIMIT $l").getOrElse(q"")
-      _ = logger.info("Requesting datasets with query")
       r <- run(q"SELECT $columns FROM $existingCollectionName WHERE $selectionPredicates $limitQuery".as[DatasetsRow])
-      parsed <- parseAll(r) ?~> "Parsing datasets failed"
+      parsed <- parseAll(r)
     } yield parsed
 
   def findAllCompactWithSearch(isActiveOpt: Option[Boolean] = None,

app/models/dataset/DatasetService.scala

@@ -349,19 +349,19 @@ class DatasetService @Inject()(organizationDAO: OrganizationDAO,
         organizationDAO.findOne(dataset._organization) ?~> "organization.notFound"
       }
       dataStore <- Fox.fillOption(dataStore) {
-        dataStoreFor(dataset) ?~> s"fetching data store failed for dataset ${dataset._id}"
+        dataStoreFor(dataset) ?~> "dataStore.notFound"
       }
-      teams <- teamService.allowedTeamsForDataset(dataset, cumulative = false, requestingUserOpt) ?~> "dataset.list.fetchAllowedTeamsFailed" ?~> s"for dataset ${dataset._id}"
-      teamsJs <- Fox.serialCombined(teams)(t => teamService.publicWrites(t, Some(organization))) ?~> "dataset.list.teamWritesFailed" ?~> s"for dataset ${dataset._id}"
-      teamsCumulative <- teamService.allowedTeamsForDataset(dataset, cumulative = true, requestingUserOpt) ?~> "dataset.list.fetchAllowedTeamsFailed" ?~> s"for dataset ${dataset._id}"
-      teamsCumulativeJs <- Fox.serialCombined(teamsCumulative)(t => teamService.publicWrites(t, Some(organization))) ?~> "dataset.list.teamWritesFailed" ?~> s"for dataset ${dataset._id}"
-      logoUrl <- logoUrlFor(dataset, Some(organization)) ?~> "dataset.list.fetchLogoUrlFailed" ?~> s"for dataset ${dataset._id}"
-      isEditable <- isEditableBy(dataset, requestingUserOpt, requestingUserTeamManagerMemberships) ?~> "dataset.list.isEditableCheckFailed" ?~> s"for dataset ${dataset._id}"
-      lastUsedByUser <- lastUsedTimeFor(dataset._id, requestingUserOpt) ?~> "dataset.list.fetchLastUsedTimeFailed" ?~> s"for dataset ${dataset._id}"
-      dataStoreJs <- dataStoreService.publicWrites(dataStore) ?~> "dataset.list.dataStoreWritesFailed" ?~> s"for dataset ${dataset._id}"
-      dataSource <- dataSourceFor(dataset, Some(organization)) ?~> "dataset.list.fetchDataSourceFailed" ?~> s"for dataset ${dataset._id}"
+      teams <- teamService.allowedTeamsForDataset(dataset, cumulative = false, requestingUserOpt) ?~> "dataset.list.fetchAllowedTeamsFailed"
+      teamsJs <- Fox.serialCombined(teams)(t => teamService.publicWrites(t, Some(organization))) ?~> "dataset.list.teamWritesFailed"
+      teamsCumulative <- teamService.allowedTeamsForDataset(dataset, cumulative = true, requestingUserOpt) ?~> "dataset.list.fetchAllowedTeamsFailed"
+      teamsCumulativeJs <- Fox.serialCombined(teamsCumulative)(t => teamService.publicWrites(t, Some(organization))) ?~> "dataset.list.teamWritesFailed"
+      logoUrl <- logoUrlFor(dataset, Some(organization)) ?~> "dataset.list.fetchLogoUrlFailed"
+      isEditable <- isEditableBy(dataset, requestingUserOpt, requestingUserTeamManagerMemberships) ?~> "dataset.list.isEditableCheckFailed"
+      lastUsedByUser <- lastUsedTimeFor(dataset._id, requestingUserOpt) ?~> "dataset.list.fetchLastUsedTimeFailed"
+      dataStoreJs <- dataStoreService.publicWrites(dataStore) ?~> "dataset.list.dataStoreWritesFailed"
+      dataSource <- dataSourceFor(dataset, Some(organization)) ?~> "dataset.list.fetchDataSourceFailed"
       usedStorageBytes <- Fox.runIf(requestingUserOpt.exists(u => u._organization == dataset._organization))(
-        organizationDAO.getUsedStorageForDataset(dataset._id)) ?~> s"fetching used storage failed for ${dataset._id}"
+        organizationDAO.getUsedStorageForDataset(dataset._id))
     } yield {
       Json.obj(
         "id" -> dataset._id,

app/models/user/UserService.scala

@@ -279,11 +279,11 @@ class UserService @Inject()(conf: WkConf,
     userExperiencesDAO.findAllExperiencesForUser(_user)
 
   def teamMembershipsFor(_user: ObjectId): Fox[List[TeamMembership]] =
-    userDAO.findTeamMembershipsForUser(_user)
+    userDAO.findTeamMembershipsForUser(_user) ?~> "user.team.memberships.failed"
 
   def teamManagerMembershipsFor(_user: ObjectId): Fox[List[TeamMembership]] =
     for {
-      teamMemberships <- teamMembershipsFor(_user)
+      teamMemberships <- teamMembershipsFor(_user) ?~> "user.team.memberships.failed"
     } yield teamMemberships.filter(_.isTeamManager)
 
   def teamManagerTeamIdsFor(_user: ObjectId): Fox[List[ObjectId]] =