Merge branch 'master' of github.com:scalableminds/webknossos into docs

* 'master' of github.com:scalableminds/webknossos: Add option to change active segement (#6006) ship expanded testdata (#6004) Allow Download of Public Annotations While Logged Out (#6001) Increase maximum bucket count limit for segmentation layers (#6000) Allow cancelling uploads (#5958) Store layer visibility with annotation (#5967) Fix async volume bugs (#5955)
scalableminds · Feb 2, 2022 · 997527b · 997527b
2 parents 201208f + 8a0ecac
commit 997527b
Show file tree

Hide file tree

Showing 63 changed files with 1,584 additions and 506 deletions.
diff --git a/CHANGELOG.unreleased.md b/CHANGELOG.unreleased.md
@@ -11,15 +11,23 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released
 [Commits](https://github.com/scalableminds/webknossos/compare/22.02.0...HEAD)
 
 ### Added
+- Added the option to make a segment's ID active via the right-click context menu in the segments list. [#5935](https://github.com/scalableminds/webknossos/pull/6006)
 - Added a button next to the histogram which adapts the contrast and brightness to the currently visible data. [#5961](https://github.com/scalableminds/webknossos/pull/5961)
+- Running uploads can now be cancelled. [#5958](https://github.com/scalableminds/webknossos/pull/5958)
 
 ### Changed
 - Upgraded webpack build tool to v5 and all other webpack related dependencies to their latest version. Enabled persistent caching which speeds up server restarts during development as well as production builds. [#5969](https://github.com/scalableminds/webknossos/pull/5969)
+- Improved stability when quickly volume-annotating large structures. [#6000](https://github.com/scalableminds/webknossos/pull/6000)
+- The front-end API `labelVoxels` returns a promise now which fulfills as soon as the label operation was carried out. [#5955](https://github.com/scalableminds/webknossos/pull/5955)
+- When changing which layers are visible in an annotation, this setting is persisted in the annotation, so when you share it, viewers will see the same visibility configuration. [#5967](https://github.com/scalableminds/webknossos/pull/5967)
+- Downloading public annotations is now also allowed without being authenticated. [#6001](https://github.com/scalableminds/webknossos/pull/6001)
 
 ### Fixed
+- Fixed volume-related bugs which could corrupt the volume data in certain scenarios. [#5955](https://github.com/scalableminds/webknossos/pull/5955)
 - Fixed the placeholder resolution computation for anisotropic layers with missing base resolutions. [#5983](https://github.com/scalableminds/webknossos/pull/5983)
 - Fixed a bug where ad-hoc meshes were computed for a mapping, although it was disabled. [#5982](https://github.com/scalableminds/webknossos/pull/5982)
 
+
 ### Removed
 
 ### Breaking Changes
diff --git a/MIGRATIONS.unreleased.md b/MIGRATIONS.unreleased.md
@@ -9,3 +9,4 @@ User-facing changes are documented in the [changelog](CHANGELOG.released.md).
 [Commits](https://github.com/scalableminds/webknossos/compare/22.02.0...HEAD)
 
 ### Postgres Evolutions:
+- [081-annotation-viewconfiguration.sql](conf/evolutions/081-annotation-viewconfiguration.sql)
diff --git a/README.md b/README.md
@@ -179,6 +179,9 @@ yarn run lint
 # Format frontend code
 yarn run pretty
 
+# Format backend code
+yarn pretty-backend
+
 # Frontend type checking
 yarn flow
 

diff --git a/app/controllers/AnnotationController.scala b/app/controllers/AnnotationController.scala
@@ -332,11 +332,14 @@ class AnnotationController @Inject()(
           annotationService.updateTeamsForSharedAnnotation(annotation._id, List.empty)
         else Fox.successful(())
         tags = (request.body \ "tags").asOpt[List[String]]
+        viewConfiguration = (request.body \ "viewConfiguration").asOpt[JsObject]
         _ <- Fox.runOptional(name)(annotationDAO.updateName(annotation._id, _)) ?~> "annotation.edit.failed"
         _ <- Fox
           .runOptional(description)(annotationDAO.updateDescription(annotation._id, _)) ?~> "annotation.edit.failed"
         _ <- Fox.runOptional(visibility)(annotationDAO.updateVisibility(annotation._id, _)) ?~> "annotation.edit.failed"
         _ <- Fox.runOptional(tags)(annotationDAO.updateTags(annotation._id, _)) ?~> "annotation.edit.failed"
+        _ <- Fox
+          .runOptional(viewConfiguration)(vc => annotationDAO.updateViewConfiguration(annotation._id, Some(vc))) ?~> "annotation.edit.failed"
       } yield JsonOk(Messages("annotation.edit.success"))
   }
 

diff --git a/app/controllers/AnnotationIOController.scala b/app/controllers/AnnotationIOController.scala
@@ -238,11 +238,11 @@ Expects:
       skeletonVersion: Option[Long],
       volumeVersion: Option[Long],
       skipVolumeData: Option[Boolean]): Action[AnyContent] =
-    sil.SecuredAction.async { implicit request =>
+    sil.UserAwareAction.async { implicit request =>
       logger.trace(s"Requested download for annotation: $typ/$id")
       for {
         identifier <- AnnotationIdentifier.parse(typ, id)
-        _ = analyticsService.track(DownloadAnnotationEvent(request.identity, id, typ))
+        _ = request.identity.foreach(user => analyticsService.track(DownloadAnnotationEvent(user, id, typ)))
         result <- identifier.annotationType match {
           case AnnotationType.View            => Fox.failure("Cannot download View annotation")
           case AnnotationType.CompoundProject => downloadProject(id, request.identity, skipVolumeData.getOrElse(false))
@@ -263,7 +263,7 @@ Expects:
   // TODO: select versions per layer
   private def downloadExplorational(annotationId: String,
                                     typ: String,
-                                    issuingUser: User,
+                                    issuingUser: Option[User],
                                     skeletonVersion: Option[Long],
                                     volumeVersion: Option[Long],
                                     skipVolumeData: Boolean)(implicit ctx: DBAccessContext) = {
@@ -347,9 +347,11 @@ Expects:
     }
   }
 
-  private def downloadProject(projectId: String, user: User, skipVolumeData: Boolean)(implicit ctx: DBAccessContext,
-                                                                                      m: MessagesProvider) =
+  private def downloadProject(projectId: String, userOpt: Option[User], skipVolumeData: Boolean)(
+      implicit ctx: DBAccessContext,
+      m: MessagesProvider) =
     for {
+      user <- userOpt.toFox ?~> Messages("notAllowed") ~> FORBIDDEN
       projectIdValidated <- ObjectId.parse(projectId)
       project <- projectDAO.findOne(projectIdValidated) ?~> Messages("project.notFound", projectId) ~> NOT_FOUND
       _ <- Fox.assertTrue(userService.isTeamManagerOrAdminOf(user, project._team)) ?~> "notAllowed" ~> FORBIDDEN
@@ -360,15 +362,17 @@ Expects:
       Ok.sendFile(file, inline = false, fileName = _ => Some(TextUtils.normalize(project.name + "_nmls.zip")))
     }
 
-  private def downloadTask(taskId: String, user: User, skipVolumeData: Boolean)(implicit ctx: DBAccessContext,
-                                                                                m: MessagesProvider) = {
+  private def downloadTask(taskId: String, userOpt: Option[User], skipVolumeData: Boolean)(
+      implicit ctx: DBAccessContext,
+      m: MessagesProvider) = {
     def createTaskZip(task: Task): Fox[TemporaryFile] = annotationService.annotationsFor(task._id).flatMap {
       annotations =>
         val finished = annotations.filter(_.state == Finished)
         annotationService.zipAnnotations(finished, task._id.toString, skipVolumeData)
     }
 
     for {
+      user <- userOpt.toFox ?~> Messages("notAllowed") ~> FORBIDDEN
       task <- taskDAO.findOne(ObjectId(taskId)).toFox ?~> Messages("task.notFound") ~> NOT_FOUND
       project <- projectDAO.findOne(task._project) ?~> Messages("project.notFound") ~> NOT_FOUND
       _ <- Fox.assertTrue(userService.isTeamManagerOrAdminOf(user, project._team)) ?~> Messages("notAllowed") ~> FORBIDDEN
@@ -379,8 +383,9 @@ Expects:
     }
   }
 
-  private def downloadTaskType(taskTypeId: String, user: User, skipVolumeData: Boolean)(
-      implicit ctx: DBAccessContext) = {
+  private def downloadTaskType(taskTypeId: String, userOpt: Option[User], skipVolumeData: Boolean)(
+      implicit ctx: DBAccessContext,
+      m: MessagesProvider) = {
     def createTaskTypeZip(taskType: TaskType) =
       for {
         tasks <- taskDAO.findAllByTaskType(taskType._id)
@@ -393,6 +398,7 @@ Expects:
       } yield zip
 
     for {
+      user <- userOpt.toFox ?~> Messages("notAllowed") ~> FORBIDDEN
       taskTypeIdValidated <- ObjectId.parse(taskTypeId) ?~> "taskType.id.invalid"
       taskType <- taskTypeDAO.findOne(taskTypeIdValidated) ?~> "taskType.notFound" ~> NOT_FOUND
       _ <- Fox.assertTrue(userService.isTeamManagerOrAdminOf(user, taskType._team)) ?~> "notAllowed" ~> FORBIDDEN

diff --git a/app/controllers/WKRemoteDataStoreController.scala b/app/controllers/WKRemoteDataStoreController.scala
@@ -145,7 +145,7 @@ class WKRemoteDataStoreController @Inject()(
     }
   }
 
-  def deleteErroneous(name: String, key: String): Action[JsValue] = Action.async(parse.json) { implicit request =>
+  def deleteDataset(name: String, key: String): Action[JsValue] = Action.async(parse.json) { implicit request =>
     dataStoreService.validateAccess(name, key) { _ =>
       for {
         datasourceId <- request.body.validate[DataSourceId].asOpt.toFox ?~> "dataStore.upload.invalid"

diff --git a/app/controllers/WKRemoteWorkerController.scala b/app/controllers/WKRemoteWorkerController.scala
@@ -51,6 +51,7 @@ class WKRemoteWorkerController @Inject()(jobDAO: JobDAO, jobService: JobService,
         _ <- jobDAO.updateStatus(jobIdParsed, request.body)
         jobAfterChange <- jobDAO.findOne(jobIdParsed)(GlobalAccessContext)
         _ = jobService.trackStatusChange(jobBeforeChange, jobAfterChange)
+        _ <- jobService.cleanUpIfFailed(jobAfterChange)
       } yield Ok
   }
 

diff --git a/app/models/annotation/Annotation.scala b/app/models/annotation/Annotation.scala
@@ -1,7 +1,7 @@
 package models.annotation
 
 import com.scalableminds.util.accesscontext.DBAccessContext
-import com.scalableminds.util.tools.{Fox, FoxImplicits}
+import com.scalableminds.util.tools.{Fox, FoxImplicits, JsonHelper}
 import com.scalableminds.webknossos.schema.Tables._
 import com.scalableminds.webknossos.tracingstore.tracings.TracingType
 import javax.inject.Inject
@@ -27,6 +27,7 @@ case class Annotation(
     description: String = "",
     visibility: AnnotationVisibility.Value = AnnotationVisibility.Internal,
     name: String = "",
+    viewConfiguration: Option[JsObject] = None,
     state: AnnotationState.Value = Active,
     statistics: JsObject = Json.obj(),
     tags: Set[String] = Set.empty,
@@ -147,6 +148,7 @@ class AnnotationDAO @Inject()(sqlClient: SQLClient, annotationLayerDAO: Annotati
     for {
       state <- AnnotationState.fromString(r.state).toFox
       typ <- AnnotationType.fromString(r.typ).toFox
+      viewconfigurationOpt <- Fox.runOptional(r.viewconfiguration)(JsonHelper.parseJsonToFox[JsObject](_))
       visibility <- AnnotationVisibility.fromString(r.visibility).toFox
       annotationLayers <- annotationLayerDAO.findAnnotationLayersFor(ObjectId(r._Id))
     } yield {
@@ -160,6 +162,7 @@ class AnnotationDAO @Inject()(sqlClient: SQLClient, annotationLayerDAO: Annotati
         r.description,
         visibility,
         r.name,
+        viewconfigurationOpt,
         state,
         Json.parse(r.statistics).as[JsObject],
         parseArrayTuple(r.tags).toSet,
@@ -319,11 +322,13 @@ class AnnotationDAO @Inject()(sqlClient: SQLClient, annotationLayerDAO: Annotati
   // update operations
 
   def insertOne(a: Annotation): Fox[Unit] = {
+    val viewConfigurationStr: Option[String] = a.viewConfiguration.map(Json.toJson(_).toString)
     val insertAnnotationQuery = sqlu"""
         insert into webknossos.annotations(_id, _dataSet, _task, _team, _user, description, visibility,
-                                           name, state, statistics, tags, tracingTime, typ, created, modified, isDeleted)
+                                           name, viewConfiguration, state, statistics, tags, tracingTime, typ, created, modified, isDeleted)
         values(${a._id.id}, ${a._dataSet.id}, ${a._task.map(_.id)}, ${a._team.id},
          ${a._user.id}, ${a.description}, '#${a.visibility.toString}', ${a.name},
+         #${optionLiteral(viewConfigurationStr.map(sanitize))},
          '#${a.state.toString}', '#${sanitize(a.statistics.toString)}',
          '#${writeArrayTuple(a.tags.toList.map(sanitize))}', ${a.tracingTime}, '#${a.typ.toString}',
          ${new java.sql.Timestamp(a.created)}, ${new java.sql.Timestamp(a.modified)}, ${a.isDeleted})
@@ -336,6 +341,7 @@ class AnnotationDAO @Inject()(sqlClient: SQLClient, annotationLayerDAO: Annotati
 
   // Task only, thus hard replacing tracing ids
   def updateInitialized(a: Annotation): Fox[Unit] = {
+    val viewConfigurationStr: Option[String] = a.viewConfiguration.map(Json.toJson(_).toString)
     val updateAnnotationQuery = sqlu"""
              update webknossos.annotations
              set
@@ -345,6 +351,7 @@ class AnnotationDAO @Inject()(sqlClient: SQLClient, annotationLayerDAO: Annotati
                description = ${a.description},
                visibility = '#${a.visibility.toString}',
                name = ${a.name},
+               viewConfiguration = #${optionLiteral(viewConfigurationStr.map(sanitize))},
                state = '#${a.state.toString}',
                statistics = '#${sanitize(a.statistics.toString)}',
                tags = '#${writeArrayTuple(a.tags.toList.map(sanitize))}',
@@ -444,6 +451,16 @@ class AnnotationDAO @Inject()(sqlClient: SQLClient, annotationLayerDAO: Annotati
 
   def updateUser(id: ObjectId, userId: ObjectId)(implicit ctx: DBAccessContext): Fox[Unit] =
     updateObjectIdCol(id, _._User, userId)
+
+  def updateViewConfiguration(id: ObjectId, viewConfiguration: Option[JsObject])(
+      implicit ctx: DBAccessContext): Fox[Unit] = {
+    val viewConfigurationStr: Option[String] = viewConfiguration.map(Json.toJson(_).toString)
+    for {
+      _ <- assertUpdateAccess(id)
+      _ <- run(sqlu"update webknossos.annotations set viewConfiguration = #${optionLiteral(
+        viewConfigurationStr.map(sanitize))} where _id = ${id.id}")
+    } yield ()
+  }
 }
 
 class SharedAnnotationsDAO @Inject()(annotationDAO: AnnotationDAO, sqlClient: SQLClient)(implicit ec: ExecutionContext)

diff --git a/app/models/annotation/AnnotationService.scala b/app/models/annotation/AnnotationService.scala
@@ -792,6 +792,7 @@ class AnnotationService @Inject()(
         "id" -> annotation.id,
         "name" -> annotation.name,
         "description" -> annotation.description,
+        "viewConfiguration" -> annotation.viewConfiguration,
         "typ" -> annotation.typ,
         "task" -> taskJson,
         "stats" -> annotation.statistics,

diff --git a/app/models/job/Job.scala b/app/models/job/Job.scala
@@ -1,17 +1,17 @@
 package models.job
 
 import java.sql.Timestamp
-
 import akka.actor.ActorSystem
 import com.scalableminds.util.accesscontext.{DBAccessContext, GlobalAccessContext}
 import com.scalableminds.util.geometry.BoundingBox
 import com.scalableminds.util.mvc.Formatter
 import com.scalableminds.util.tools.{Fox, FoxImplicits}
 import com.scalableminds.webknossos.schema.Tables._
 import com.typesafe.scalalogging.LazyLogging
+
 import javax.inject.Inject
 import models.analytics.{AnalyticsService, FailedJobEvent, RunJobEvent}
-import models.binary.DataStoreDAO
+import models.binary.{DataSetDAO, DataStoreDAO}
 import models.job.JobState.JobState
 import models.organization.OrganizationDAO
 import models.user.{MultiUserDAO, User, UserDAO}
@@ -257,6 +257,7 @@ class JobService @Inject()(wkConf: WkConf,
                            jobDAO: JobDAO,
                            dataStoreDAO: DataStoreDAO,
                            organizationDAO: OrganizationDAO,
+                           dataSetDAO: DataSetDAO,
                            analyticsService: AnalyticsService,
                            slackNotificationService: SlackNotificationService,
                            val lifecycle: ApplicationLifecycle,
@@ -309,6 +310,18 @@ class JobService @Inject()(wkConf: WkConf,
     ()
   }
 
+  def cleanUpIfFailed(job: Job): Fox[Unit] =
+    if (job.state == JobState.FAILURE && job.command == "convert_to_wkw") {
+      logger.info(s"WKW conversion job ${job._id} failed. Deleting dataset from the database, freeing the name...")
+      val commandArgs = job.commandArgs.value
+      for {
+        datasetName <- commandArgs.get("dataset_name").map(_.as[String]).toFox
+        organizationName <- commandArgs.get("organization_name").map(_.as[String]).toFox
+        dataset <- dataSetDAO.findOneByNameAndOrganizationName(datasetName, organizationName)(GlobalAccessContext)
+        _ <- dataSetDAO.deleteDataset(dataset._id)
+      } yield ()
+    } else Fox.successful(())
+
   def publicWrites(job: Job)(implicit ctx: DBAccessContext): Fox[JsObject] =
     for {
       owner <- userDAO.findOne(job._owner) ?~> "user.notFound"

diff --git a/conf/evolutions/081-annotation-viewconfiguration.sql b/conf/evolutions/081-annotation-viewconfiguration.sql
@@ -0,0 +1,11 @@
+START TRANSACTION;
+
+DROP VIEW webknossos.annotations_;
+
+ALTER TABLE webknossos.annotations ADD COLUMN viewConfiguration JSONB;
+
+CREATE VIEW webknossos.annotations_ AS SELECT * FROM webknossos.annotations WHERE NOT isDeleted;
+
+UPDATE webknossos.releaseInformation SET schemaVersion = 81;
+
+COMMIT TRANSACTION;
diff --git a/conf/evolutions/reversions/081-annotation-viewconfiguration.sql b/conf/evolutions/reversions/081-annotation-viewconfiguration.sql
@@ -0,0 +1,11 @@
+START TRANSACTION;
+
+DROP VIEW webknossos.annotations_;
+
+ALTER TABLE webknossos.annotations DROP COLUMN viewConfiguration;
+
+CREATE VIEW webknossos.annotations_ AS SELECT * FROM webknossos.annotations WHERE NOT isDeleted;
+
+UPDATE webknossos.releaseInformation SET schemaVersion = 80;
+
+COMMIT TRANSACTION;
diff --git a/conf/messages b/conf/messages
@@ -129,10 +129,13 @@ dataset.initialTeams.invalidTeams=Can only assign teams of user
 dataset.initialTeams.timeout=Timeout while setting initial teams. Was the request sent manually? Received at {0}, dataset created at {1}
 dataSet.uploader.notEmpty=Dataset already has non-empty uploader
 dataset.delete.disabled=Dataset deletion is disabled for this webKnossos instance
+dataSet.delete.webknossos.failed=Could not delete dataset from webKnossos database
+dataSet.delete.failed=Could not delete the dataset on disk.
 dataSet.upload.Datastore.restricted=Your organization is not allowed to upload datasets to this datastore. Please choose another datastore.
 dataSet.upload.validation.failed=Failed to validate Dataset information for upload.
 dataSet.upload.linkPublicOnly=Only layers of existing public datasets can be linked
 dataSet.upload.invalidLinkedLayers=Could not link all requested layers
+dataSet.upload.cancel.failed=Could not cancel the upload.
 
 dataSource.notFound=Datasource not found on datastore server
 

diff --git a/conf/webknossos.latest.routes b/conf/webknossos.latest.routes
@@ -87,7 +87,7 @@ PUT           /datastores/:name/datasources                                    c
 PATCH         /datastores/:name/status                                         controllers.WKRemoteDataStoreController.statusUpdate(name: String, key: String)
 POST          /datastores/:name/verifyUpload                                   controllers.WKRemoteDataStoreController.validateDataSetUpload(name: String, key: String, token: String)
 POST          /datastores/:name/reportDatasetUpload                            controllers.WKRemoteDataStoreController.reportDatasetUpload(name: String, key: String, token: String, dataSetName: String, dataSetSizeBytes: Long)
-POST          /datastores/:name/deleteErroneous                                controllers.WKRemoteDataStoreController.deleteErroneous(name: String, key: String)
+POST          /datastores/:name/deleteDataset                                  controllers.WKRemoteDataStoreController.deleteDataset(name: String, key: String)
 GET           /datastores/:name/jobExportProperties                            controllers.WKRemoteDataStoreController.jobExportProperties(name: String, key: String, jobId: String)
 POST          /datastores/:name/validateUserAccess                             controllers.UserTokenController.validateAccessViaDatastore(name: String, key: String, token: Option[String])
 POST          /datastores                                                      controllers.DataStoreController.create
Original file line number	Diff line number	Diff line change
Expand Up		@@ -9,3 +9,4 @@ User-facing changes are documented in the [changelog](CHANGELOG.released.md).
		[Commits](https://github.com/scalableminds/webknossos/compare/22.02.0...HEAD)

		### Postgres Evolutions:
		- [081-annotation-viewconfiguration.sql](conf/evolutions/081-annotation-viewconfiguration.sql)