From 9e4736d839b167db642a63cd4029ac75f1a90cfa Mon Sep 17 00:00:00 2001 From: frcroth Date: Tue, 4 Apr 2023 13:38:25 +0200 Subject: [PATCH 01/13] Write wkurl into nml file --- app/controllers/AnnotationIOController.scala | 25 +++++++++++-------- app/models/annotation/AnnotationService.scala | 6 +++-- app/models/annotation/nml/NmlWriter.scala | 9 +++++++ 3 files changed, 28 insertions(+), 12 deletions(-) diff --git a/app/controllers/AnnotationIOController.scala b/app/controllers/AnnotationIOController.scala index cefd27c32d..ef542bcd9a 100755 --- a/app/controllers/AnnotationIOController.scala +++ b/app/controllers/AnnotationIOController.scala @@ -43,7 +43,7 @@ import play.api.i18n.{Messages, MessagesProvider} import play.api.libs.Files.{TemporaryFile, TemporaryFileCreator} import play.api.libs.json.Json import play.api.mvc.{Action, AnyContent, MultipartFormData} -import utils.ObjectId +import utils.{ObjectId, WkConf} import scala.concurrent.{ExecutionContext, Future} @@ -62,6 +62,7 @@ class AnnotationIOController @Inject()( temporaryFileCreator: TemporaryFileCreator, annotationService: AnnotationService, analyticsService: AnalyticsService, + conf: WkConf, sil: Silhouette[WkEnv], provider: AnnotationInformationProvider, annotationUploadService: AnnotationUploadService)(implicit ec: ExecutionContext, val materializer: Materializer) @@ -370,6 +371,7 @@ Expects: dataSet.scale, None, organizationName, + conf.Http.uri, dataSet.name, Some(user), taskOpt) @@ -395,15 +397,18 @@ Expects: } user <- userService.findOneById(annotation._user, useCache = true) taskOpt <- Fox.runOptional(annotation._task)(taskDAO.findOne) - nmlStream = nmlWriter.toNmlStream(fetchedSkeletonLayers ::: fetchedVolumeLayers, - Some(annotation), - dataset.scale, - None, - organizationName, - dataset.name, - Some(user), - taskOpt, - skipVolumeData) + nmlStream = nmlWriter.toNmlStream( + fetchedSkeletonLayers ::: fetchedVolumeLayers, + Some(annotation), + dataset.scale, + None, + organizationName, + conf.Http.uri, + dataset.name, + Some(user), + taskOpt, + skipVolumeData + ) temporaryFile = temporaryFileCreator.create() zipper = ZipIO.startZip(new BufferedOutputStream(new FileOutputStream(new File(temporaryFile.path.toString)))) _ <- zipper.addFileFromEnumerator(name + ".nml", nmlStream) diff --git a/app/models/annotation/AnnotationService.scala b/app/models/annotation/AnnotationService.scala index 07e55242b6..4e05b06cfb 100755 --- a/app/models/annotation/AnnotationService.scala +++ b/app/models/annotation/AnnotationService.scala @@ -52,7 +52,7 @@ import play.api.i18n.{Messages, MessagesProvider} import play.api.libs.Files.{TemporaryFile, TemporaryFileCreator} import play.api.libs.iteratee.Enumerator import play.api.libs.json.{JsNull, JsObject, JsValue, Json} -import utils.ObjectId +import utils.{ObjectId, WkConf} import java.io.{BufferedOutputStream, File, FileOutputStream} import javax.inject.Inject @@ -104,7 +104,8 @@ class AnnotationService @Inject()( nmlWriter: NmlWriter, temporaryFileCreator: TemporaryFileCreator, meshDAO: MeshDAO, - meshService: MeshService + meshService: MeshService, + conf: WkConf, )(implicit ec: ExecutionContext, val materializer: Materializer) extends BoxImplicits with FoxImplicits @@ -639,6 +640,7 @@ class AnnotationService @Inject()( scaleOpt, Some(name + "_data.zip"), organizationName, + conf.Http.uri, datasetName, Some(user), taskOpt) diff --git a/app/models/annotation/nml/NmlWriter.scala b/app/models/annotation/nml/NmlWriter.scala index 9ff95f17d5..b09ca2da6d 100644 --- a/app/models/annotation/nml/NmlWriter.scala +++ b/app/models/annotation/nml/NmlWriter.scala @@ -22,6 +22,7 @@ case class NmlParameters( dataSetName: String, organizationName: String, description: Option[String], + wkUrl: String, scale: Option[Vec3Double], createdTimestamp: Long, editPosition: Vec3IntProto, @@ -41,6 +42,7 @@ class NmlWriter @Inject()(implicit ec: ExecutionContext) extends FoxImplicits { volumeFilename: Option[String], organizationName: String, datasetName: String, + wkUrl: String, annotationOwner: Option[User], annotationTask: Option[Task], skipVolumeData: Boolean = false): Enumerator[Array[Byte]] = Enumerator.outputStream { os => @@ -53,6 +55,7 @@ class NmlWriter @Inject()(implicit ec: ExecutionContext) extends FoxImplicits { scale, volumeFilename, organizationName, + wkUrl, datasetName, annotationOwner, annotationTask, @@ -66,6 +69,7 @@ class NmlWriter @Inject()(implicit ec: ExecutionContext) extends FoxImplicits { scale: Option[Vec3Double], volumeFilename: Option[String], organizationName: String, + wkUrl: String, datasetName: String, annotationOwner: Option[User], annotationTask: Option[Task], @@ -82,6 +86,7 @@ class NmlWriter @Inject()(implicit ec: ExecutionContext) extends FoxImplicits { volumeLayers, annotation: Option[Annotation], organizationName, + wkUrl, datasetName, scale) _ = writeParameters(parameters) @@ -103,6 +108,7 @@ class NmlWriter @Inject()(implicit ec: ExecutionContext) extends FoxImplicits { volumeLayers: List[FetchedAnnotationLayer], annotation: Option[Annotation], organizationName: String, + wkUrl: String, datasetName: String, scale: Option[Vec3Double]): Fox[NmlParameters] = for { @@ -113,6 +119,7 @@ class NmlWriter @Inject()(implicit ec: ExecutionContext) extends FoxImplicits { datasetName, organizationName, annotation.map(_.description), + wkUrl, scale, s.createdTimestamp, s.editPosition, @@ -127,6 +134,7 @@ class NmlWriter @Inject()(implicit ec: ExecutionContext) extends FoxImplicits { datasetName, organizationName, annotation.map(_.description), + wkUrl, scale, v.createdTimestamp, v.editPosition, @@ -154,6 +162,7 @@ class NmlWriter @Inject()(implicit ec: ExecutionContext) extends FoxImplicits { writer.writeAttribute("name", parameters.dataSetName) writer.writeAttribute("organization", parameters.organizationName) parameters.description.foreach(writer.writeAttribute("description", _)) + writer.writeAttribute("wkUrl", parameters.wkUrl) } Xml.withinElementSync("scale") { writer.writeAttribute("x", parameters.scale.map(_.x).getOrElse(-1).toString) From 9538eb8a3f8cf3f5942e42be432d100a3ded88bd Mon Sep 17 00:00:00 2001 From: frcroth Date: Tue, 4 Apr 2023 15:04:09 +0200 Subject: [PATCH 02/13] Check wk url for better errors on annotation uploading --- app/controllers/AnnotationIOController.scala | 33 ++++++++++++++----- .../annotation/AnnotationUploadService.scala | 12 +++---- app/models/annotation/nml/NmlParser.scala | 9 +++-- app/models/annotation/nml/NmlResults.scala | 5 ++- app/models/annotation/nml/NmlWriter.scala | 2 +- conf/messages | 2 ++ test/backend/NMLUnitTestSuite.scala | 26 +++++++++++---- 7 files changed, 64 insertions(+), 25 deletions(-) diff --git a/app/controllers/AnnotationIOController.scala b/app/controllers/AnnotationIOController.scala index ef542bcd9a..6dffcf9c47 100755 --- a/app/controllers/AnnotationIOController.scala +++ b/app/controllers/AnnotationIOController.scala @@ -103,9 +103,9 @@ Expects: val attachedFiles = request.body.files.map(f => (f.ref.path.toFile, f.filename)) val parsedFiles = annotationUploadService.extractFromFiles(attachedFiles, useZipName = true, overwritingDataSetName) - val parsedFilesWraped = + val parsedFilesWrapped = annotationUploadService.wrapOrPrefixTrees(parsedFiles.parseResults, shouldCreateGroupForEachFile) - val parseResultsFiltered: List[NmlParseResult] = parsedFilesWraped.filter(_.succeeded) + val parseResultsFiltered: List[NmlParseResult] = parsedFilesWrapped.filter(_.succeeded) if (parseResultsFiltered.isEmpty) { returnError(parsedFiles) @@ -114,13 +114,15 @@ Expects: parseSuccesses <- Fox.serialCombined(parseResultsFiltered)(r => r.toSuccessBox) name = nameForUploaded(parseResultsFiltered.map(_.fileName)) description = descriptionForNMLs(parseResultsFiltered.map(_.description)) + wkUrl = wkUrlsForNMLs(parseResultsFiltered.map(_.wkUrl)) _ <- assertNonEmpty(parseSuccesses) skeletonTracings = parseSuccesses.flatMap(_.skeletonTracing) // Create a list of volume layers for each uploaded (non-skeleton-only) annotation. // This is what determines the merging strategy for volume layers volumeLayersGroupedRaw = parseSuccesses.map(_.volumeLayers).filter(_.nonEmpty) dataSet <- findDataSetForUploadedAnnotations(skeletonTracings, - volumeLayersGroupedRaw.flatten.map(_.tracing)) + volumeLayersGroupedRaw.flatten.map(_.tracing), + wkUrl) volumeLayersGrouped <- adaptVolumeTracingsToFallbackLayer(volumeLayersGroupedRaw, dataSet) tracingStoreClient <- tracingStoreService.clientFor(dataSet) mergedVolumeLayers <- mergeAndSaveVolumeLayers(volumeLayersGrouped, @@ -199,19 +201,31 @@ Expects: private def findDataSetForUploadedAnnotations( skeletonTracings: List[SkeletonTracing], - volumeTracings: List[VolumeTracing])(implicit mp: MessagesProvider, ctx: DBAccessContext): Fox[DataSet] = + volumeTracings: List[VolumeTracing], + wkUrl: String)(implicit mp: MessagesProvider, ctx: DBAccessContext): Fox[DataSet] = for { dataSetName <- assertAllOnSameDataSet(skeletonTracings, volumeTracings) ?~> "nml.file.differentDatasets" organizationNameOpt <- assertAllOnSameOrganization(skeletonTracings, volumeTracings) ?~> "nml.file.differentDatasets" organizationIdOpt <- Fox.runOptional(organizationNameOpt) { organizationDAO.findOneByName(_)(GlobalAccessContext).map(_._id) - } ?~> Messages("organization.notFound", organizationNameOpt.getOrElse("")) ~> NOT_FOUND + } ?~> (if (wkUrl.nonEmpty && conf.Http.uri != wkUrl) { + Messages("organization.notFound.wrongHost", organizationNameOpt.getOrElse(""), wkUrl, conf.Http.uri) + } else { Messages("organization.notFound", organizationNameOpt.getOrElse("")) }) ~> + NOT_FOUND organizationId <- Fox.fillOption(organizationIdOpt) { dataSetDAO.getOrganizationForDataSet(dataSetName)(GlobalAccessContext) } ?~> Messages("dataSet.noAccess", dataSetName) ~> FORBIDDEN - dataSet <- dataSetDAO.findOneByNameAndOrganization(dataSetName, organizationId) ?~> Messages( - "dataSet.noAccess", - dataSetName) ~> FORBIDDEN + dataSet <- dataSetDAO.findOneByNameAndOrganization(dataSetName, organizationId) ?~> (if (wkUrl.nonEmpty && conf.Http.uri != wkUrl) { + Messages( + "dataSet.noAccess.wrongHost", + dataSetName, + wkUrl, + conf.Http.uri) + } else { + Messages( + "dataSet.noAccess", + dataSetName) + }) ~> FORBIDDEN } yield dataSet private def nameForUploaded(fileNames: Seq[String]) = @@ -223,6 +237,9 @@ Expects: private def descriptionForNMLs(descriptions: Seq[Option[String]]) = if (descriptions.size == 1) descriptions.headOption.flatten.getOrElse("") else "" + private def wkUrlsForNMLs(wkUrls: Seq[Option[String]]) = + if (wkUrls.size == 1) wkUrls.headOption.flatten.getOrElse("") else "" + private def returnError(zipParseResult: NmlResults.MultiNmlParseResult)(implicit messagesProvider: MessagesProvider) = if (zipParseResult.containsFailure) { val errors = zipParseResult.parseResults.flatMap { diff --git a/app/models/annotation/AnnotationUploadService.scala b/app/models/annotation/AnnotationUploadService.scala index 90f16e81a6..29a113fbf7 100644 --- a/app/models/annotation/AnnotationUploadService.scala +++ b/app/models/annotation/AnnotationUploadService.scala @@ -38,8 +38,8 @@ class AnnotationUploadService @Inject()(tempFileService: TempFileService) extend isTaskUpload: Boolean, basePath: Option[String] = None)(implicit m: MessagesProvider): NmlParseResult = NmlParser.parse(name, inputStream, overwritingDataSetName, isTaskUpload, basePath) match { - case Full((skeletonTracing, uploadedVolumeLayers, description)) => - NmlParseSuccess(name, skeletonTracing, uploadedVolumeLayers, description) + case Full((skeletonTracing, uploadedVolumeLayers, description, wkUrl)) => + NmlParseSuccess(name, skeletonTracing, uploadedVolumeLayers, description, wkUrl) case Failure(msg, _, chain) => NmlParseFailure(name, msg + chain.map(_ => formatChain(chain)).getOrElse("")) case Empty => NmlParseEmpty(name) } @@ -82,8 +82,8 @@ class AnnotationUploadService @Inject()(tempFileService: TempFileService) extend if (parseResults.length > 1) { parseResults.map { - case NmlParseSuccess(name, Some(skeletonTracing), uploadedVolumeLayers, description) => - NmlParseSuccess(name, Some(renameTrees(name, skeletonTracing)), uploadedVolumeLayers, description) + case NmlParseSuccess(name, Some(skeletonTracing), uploadedVolumeLayers, description, wkUrl) => + NmlParseSuccess(name, Some(renameTrees(name, skeletonTracing)), uploadedVolumeLayers, description, wkUrl) case r => r } } else { @@ -104,8 +104,8 @@ class AnnotationUploadService @Inject()(tempFileService: TempFileService) extend } parseResults.map { - case NmlParseSuccess(name, Some(skeletonTracing), uploadedVolumeLayers, description) => - NmlParseSuccess(name, Some(wrapTreesInGroup(name, skeletonTracing)), uploadedVolumeLayers, description) + case NmlParseSuccess(name, Some(skeletonTracing), uploadedVolumeLayers, description, wkUrl) => + NmlParseSuccess(name, Some(wrapTreesInGroup(name, skeletonTracing)), uploadedVolumeLayers, description, wkUrl) case r => r } } diff --git a/app/models/annotation/nml/NmlParser.scala b/app/models/annotation/nml/NmlParser.scala index abffe99867..3030a7557e 100755 --- a/app/models/annotation/nml/NmlParser.scala +++ b/app/models/annotation/nml/NmlParser.scala @@ -26,6 +26,7 @@ object NmlParser extends LazyLogging with ProtoGeometryImplicits with ColorGener private val DEFAULT_RESOLUTION = 0 private val DEFAULT_BITDEPTH = 0 private val DEFAULT_DESCRIPTION = "" + private val DEFAULT_WKURL = "" private val DEFAULT_INTERPOLATION = false private val DEFAULT_TIMESTAMP = 0L @@ -34,7 +35,7 @@ object NmlParser extends LazyLogging with ProtoGeometryImplicits with ColorGener overwritingDataSetName: Option[String], isTaskUpload: Boolean, basePath: Option[String] = None)( - implicit m: MessagesProvider): Box[(Option[SkeletonTracing], List[UploadedVolumeLayer], String)] = + implicit m: MessagesProvider): Box[(Option[SkeletonTracing], List[UploadedVolumeLayer], String, String)] = try { val data = XML.load(nmlInputStream) for { @@ -52,6 +53,7 @@ object NmlParser extends LazyLogging with ProtoGeometryImplicits with ColorGener } yield { val dataSetName = overwritingDataSetName.getOrElse(parseDataSetName(parameters \ "experiment")) val description = parseDescription(parameters \ "experiment") + val wkUrl = parseWkUrl(parameters \ "experiment") val organizationName = if (overwritingDataSetName.isDefined) None else parseOrganizationName(parameters \ "experiment") val activeNodeId = parseActiveNode(parameters \ "activeNode") @@ -115,7 +117,7 @@ object NmlParser extends LazyLogging with ProtoGeometryImplicits with ColorGener ) ) - (skeletonTracingOpt, volumeLayers, description) + (skeletonTracingOpt, volumeLayers, description, wkUrl) } } catch { case e: org.xml.sax.SAXParseException if e.getMessage.startsWith("Premature end of file") => @@ -232,6 +234,9 @@ object NmlParser extends LazyLogging with ProtoGeometryImplicits with ColorGener private def parseDescription(nodes: NodeSeq): String = nodes.headOption.map(node => getSingleAttribute(node, "description")).getOrElse(DEFAULT_DESCRIPTION) + private def parseWkUrl(nodes: NodeSeq): String = + nodes.headOption.map(node => getSingleAttribute(node, "wkUrl")).getOrElse(DEFAULT_WKURL) + private def parseOrganizationName(nodes: NodeSeq): Option[String] = nodes.headOption.flatMap(node => getSingleAttributeOpt(node, "organization")) diff --git a/app/models/annotation/nml/NmlResults.scala b/app/models/annotation/nml/NmlResults.scala index e4fbf08966..eb08dda9c7 100644 --- a/app/models/annotation/nml/NmlResults.scala +++ b/app/models/annotation/nml/NmlResults.scala @@ -14,6 +14,7 @@ object NmlResults extends LazyLogging { def fileName: String def description: Option[String] = None + def wkUrl: Option[String] = None def succeeded: Boolean @@ -32,11 +33,13 @@ object NmlResults extends LazyLogging { case class NmlParseSuccess(fileName: String, skeletonTracing: Option[SkeletonTracing], volumeLayers: List[UploadedVolumeLayer], - _description: String) + _description: String, + _wkUrl: String) extends NmlParseResult { def succeeded = true override def description: Option[String] = Some(_description) + override def wkUrl: Option[String] = Some(_wkUrl) override def withName(name: String): NmlParseResult = this.copy(fileName = name) } diff --git a/app/models/annotation/nml/NmlWriter.scala b/app/models/annotation/nml/NmlWriter.scala index b09ca2da6d..a6fa66dea2 100644 --- a/app/models/annotation/nml/NmlWriter.scala +++ b/app/models/annotation/nml/NmlWriter.scala @@ -41,8 +41,8 @@ class NmlWriter @Inject()(implicit ec: ExecutionContext) extends FoxImplicits { scale: Option[Vec3Double], volumeFilename: Option[String], organizationName: String, - datasetName: String, wkUrl: String, + datasetName: String, annotationOwner: Option[User], annotationTask: Option[Task], skipVolumeData: Boolean = false): Enumerator[Array[Byte]] = Enumerator.outputStream { os => diff --git a/conf/messages b/conf/messages index d3f99045ec..cc5737c8c0 100644 --- a/conf/messages +++ b/conf/messages @@ -33,6 +33,7 @@ team.inUse.projects=Team is referenced by {0} projects organization.create.forbidden=You are not allowed to create a new organization organization.create.failed=Failed to create a new organization organization.notFound=Organization {0} could not be found +organization.notFound.wrongHost=Organization {0} could not be found. Please check whether you are on the correct WEBKNOSSOS instance. The uploaded file indicates {1} while this instance is {2}. organization.list.failed=Failed to retrieve list of organizations. organization.name.invalid=This organization name contains illegal characters. Please only use letters and numbers. organization.name.alreadyInUse=This name is already claimed by a different organization and not available anymore. Please choose a different name. @@ -75,6 +76,7 @@ dataSet.notFound=Dataset {0} does not exist or could not be accessed dataSet.notFoundConsiderLogin=Dataset {0} does not exist or could not be accessed. You may need to log in. dataSet.notFoundForAnnotation=The Dataset for this annotation does not exist or could not be accessed. dataSet.noAccess=Could not access DataSet {0}. Does your team have access? +dataSet.noAccess.wrongHost=Could not access DataSet {0}. Please check whether you are on the correct WEBKNOSSOS instance. The uploaded file indicates {1} while this instance is {2}. dataSet.noAccessById=Could not access the corresponding DataSet. This is likely because you are not a member of a team that has access to it. dataSet.notImported=Dataset {0} is not imported dataSet.name.invalid.characters=Dataset name is invalid. Please use only letters, digits, dots, underscores, hypens. diff --git a/test/backend/NMLUnitTestSuite.scala b/test/backend/NMLUnitTestSuite.scala index 4bd1848768..653d5d3c9d 100644 --- a/test/backend/NMLUnitTestSuite.scala +++ b/test/backend/NMLUnitTestSuite.scala @@ -22,22 +22,34 @@ class NMLUnitTestSuite extends PlaySpec { override def messages: Messages = m.preferred({ FakeRequest("GET", "/") }) } - def writeAndParseTracing(skeletonTracing: SkeletonTracing) - : Box[(Option[SkeletonTracing], List[UploadedVolumeLayer], String)] = { - val annotationLayers = List(FetchedAnnotationLayer("dummySkeletonTracingId", AnnotationLayer.defaultSkeletonLayerName, Left(skeletonTracing), None)) + def writeAndParseTracing( + skeletonTracing: SkeletonTracing): Box[(Option[SkeletonTracing], List[UploadedVolumeLayer], String, String)] = { + val annotationLayers = List( + FetchedAnnotationLayer("dummySkeletonTracingId", + AnnotationLayer.defaultSkeletonLayerName, + Left(skeletonTracing), + None)) val nmlEnumarator = - new NmlWriter().toNmlStream(annotationLayers, None, None, None, "testOrganization", "dummy_dataset", None, None) + new NmlWriter().toNmlStream(annotationLayers, + None, + None, + None, + "testOrganization", + "http://wk.test", + "dummy_dataset", + None, + None) val arrayFuture = Iteratee.flatten(nmlEnumarator |>> Iteratee.consume[Array[Byte]]()).run val array = Await.result(arrayFuture, Duration.Inf) NmlParser.parse("", new ByteArrayInputStream(array), None, isTaskUpload = true) } def isParseSuccessful( - parsedTracing: Box[(Option[SkeletonTracing], List[UploadedVolumeLayer], String)]): Boolean = + parsedTracing: Box[(Option[SkeletonTracing], List[UploadedVolumeLayer], String, String)]): Boolean = parsedTracing match { case Full(tuple) => tuple match { - case (Some(_), _, _) => true + case (Some(_), _, _, _) => true case _ => false } case _ => false @@ -50,7 +62,7 @@ class NMLUnitTestSuite extends PlaySpec { writeAndParseTracing(dummyTracing) match { case Full(tuple) => tuple match { - case (Some(tracing), _, _) => + case (Some(tracing), _, _, _) => assert(tracing == dummyTracing) case _ => throw new Exception } From 2e5830b1993a64b71c7a5a64a7dec969a14eee16 Mon Sep 17 00:00:00 2001 From: frcroth Date: Wed, 5 Apr 2023 11:16:57 +0200 Subject: [PATCH 03/13] Fix typo --- test/backend/NMLUnitTestSuite.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/backend/NMLUnitTestSuite.scala b/test/backend/NMLUnitTestSuite.scala index 653d5d3c9d..53237df954 100644 --- a/test/backend/NMLUnitTestSuite.scala +++ b/test/backend/NMLUnitTestSuite.scala @@ -29,7 +29,7 @@ class NMLUnitTestSuite extends PlaySpec { AnnotationLayer.defaultSkeletonLayerName, Left(skeletonTracing), None)) - val nmlEnumarator = + val nmlEnumerator = new NmlWriter().toNmlStream(annotationLayers, None, None, @@ -39,7 +39,7 @@ class NMLUnitTestSuite extends PlaySpec { "dummy_dataset", None, None) - val arrayFuture = Iteratee.flatten(nmlEnumarator |>> Iteratee.consume[Array[Byte]]()).run + val arrayFuture = Iteratee.flatten(nmlEnumerator |>> Iteratee.consume[Array[Byte]]()).run val array = Await.result(arrayFuture, Duration.Inf) NmlParser.parse("", new ByteArrayInputStream(array), None, isTaskUpload = true) } From 95bd2962a212884800c3945260e7416e2c326dec Mon Sep 17 00:00:00 2001 From: frcroth Date: Wed, 5 Apr 2023 11:18:50 +0200 Subject: [PATCH 04/13] Update changelog --- CHANGELOG.unreleased.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.unreleased.md b/CHANGELOG.unreleased.md index 65ca4a145c..237f2b48c9 100644 --- a/CHANGELOG.unreleased.md +++ b/CHANGELOG.unreleased.md @@ -19,6 +19,7 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released ### Changed - Moved the view mode selection in the toolbar next to the position field. [#6949](https://github.com/scalableminds/webknossos/pull/6949) +- When saving annotations, the URL of the webknossos instance is stored in the resulting NML file. [#6964](https://github.com/scalableminds/webknossos/pull/6964) ### Fixed - Fixed incorrect initial tab when clicking "Show Annotations" for a user in the user list. Also, the datasets tab was removed from that page as it was the same as the datasets table from the main dashboard. [#6957](https://github.com/scalableminds/webknossos/pull/6957) From 7e5bebc35d4483648f8bb9cd720df6e6dc694775 Mon Sep 17 00:00:00 2001 From: frcroth Date: Wed, 12 Apr 2023 10:04:40 +0200 Subject: [PATCH 05/13] Apply suggestions from code review --- app/controllers/AnnotationIOController.scala | 2 +- app/models/annotation/nml/NmlParser.scala | 7 +++---- app/models/annotation/nml/NmlResults.scala | 4 ++-- test/backend/NMLUnitTestSuite.scala | 4 ++-- 4 files changed, 8 insertions(+), 9 deletions(-) diff --git a/app/controllers/AnnotationIOController.scala b/app/controllers/AnnotationIOController.scala index 6dffcf9c47..c8c071b6e6 100755 --- a/app/controllers/AnnotationIOController.scala +++ b/app/controllers/AnnotationIOController.scala @@ -238,7 +238,7 @@ Expects: if (descriptions.size == 1) descriptions.headOption.flatten.getOrElse("") else "" private def wkUrlsForNMLs(wkUrls: Seq[Option[String]]) = - if (wkUrls.size == 1) wkUrls.headOption.flatten.getOrElse("") else "" + if (wkUrls.toSet.size == 1) wkUrls.headOption.flatten.getOrElse("") else "" private def returnError(zipParseResult: NmlResults.MultiNmlParseResult)(implicit messagesProvider: MessagesProvider) = if (zipParseResult.containsFailure) { diff --git a/app/models/annotation/nml/NmlParser.scala b/app/models/annotation/nml/NmlParser.scala index 3030a7557e..1d417d6b2e 100755 --- a/app/models/annotation/nml/NmlParser.scala +++ b/app/models/annotation/nml/NmlParser.scala @@ -26,7 +26,6 @@ object NmlParser extends LazyLogging with ProtoGeometryImplicits with ColorGener private val DEFAULT_RESOLUTION = 0 private val DEFAULT_BITDEPTH = 0 private val DEFAULT_DESCRIPTION = "" - private val DEFAULT_WKURL = "" private val DEFAULT_INTERPOLATION = false private val DEFAULT_TIMESTAMP = 0L @@ -35,7 +34,7 @@ object NmlParser extends LazyLogging with ProtoGeometryImplicits with ColorGener overwritingDataSetName: Option[String], isTaskUpload: Boolean, basePath: Option[String] = None)( - implicit m: MessagesProvider): Box[(Option[SkeletonTracing], List[UploadedVolumeLayer], String, String)] = + implicit m: MessagesProvider): Box[(Option[SkeletonTracing], List[UploadedVolumeLayer], String, Option[String])] = try { val data = XML.load(nmlInputStream) for { @@ -234,8 +233,8 @@ object NmlParser extends LazyLogging with ProtoGeometryImplicits with ColorGener private def parseDescription(nodes: NodeSeq): String = nodes.headOption.map(node => getSingleAttribute(node, "description")).getOrElse(DEFAULT_DESCRIPTION) - private def parseWkUrl(nodes: NodeSeq): String = - nodes.headOption.map(node => getSingleAttribute(node, "wkUrl")).getOrElse(DEFAULT_WKURL) + private def parseWkUrl(nodes: NodeSeq): Option[String] = + nodes.headOption.map(node => getSingleAttribute(node, "wkUrl")) private def parseOrganizationName(nodes: NodeSeq): Option[String] = nodes.headOption.flatMap(node => getSingleAttributeOpt(node, "organization")) diff --git a/app/models/annotation/nml/NmlResults.scala b/app/models/annotation/nml/NmlResults.scala index eb08dda9c7..a89b6ffac4 100644 --- a/app/models/annotation/nml/NmlResults.scala +++ b/app/models/annotation/nml/NmlResults.scala @@ -34,12 +34,12 @@ object NmlResults extends LazyLogging { skeletonTracing: Option[SkeletonTracing], volumeLayers: List[UploadedVolumeLayer], _description: String, - _wkUrl: String) + _wkUrl: Option[String]) extends NmlParseResult { def succeeded = true override def description: Option[String] = Some(_description) - override def wkUrl: Option[String] = Some(_wkUrl) + override def wkUrl: Option[String] = _wkUrl override def withName(name: String): NmlParseResult = this.copy(fileName = name) } diff --git a/test/backend/NMLUnitTestSuite.scala b/test/backend/NMLUnitTestSuite.scala index 53237df954..637ec97bae 100644 --- a/test/backend/NMLUnitTestSuite.scala +++ b/test/backend/NMLUnitTestSuite.scala @@ -22,8 +22,8 @@ class NMLUnitTestSuite extends PlaySpec { override def messages: Messages = m.preferred({ FakeRequest("GET", "/") }) } - def writeAndParseTracing( - skeletonTracing: SkeletonTracing): Box[(Option[SkeletonTracing], List[UploadedVolumeLayer], String, String)] = { + def writeAndParseTracing(skeletonTracing: SkeletonTracing) + : Box[(Option[SkeletonTracing], List[UploadedVolumeLayer], String, Option[String])] = { val annotationLayers = List( FetchedAnnotationLayer("dummySkeletonTracingId", AnnotationLayer.defaultSkeletonLayerName, From ea50b52360556ef7fb2260d6095756c7d23a5c5d Mon Sep 17 00:00:00 2001 From: frcroth Date: Wed, 12 Apr 2023 10:08:07 +0200 Subject: [PATCH 06/13] Write wkurl in front end nml serializer --- frontend/javascripts/oxalis/model/helpers/nml_helpers.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/frontend/javascripts/oxalis/model/helpers/nml_helpers.ts b/frontend/javascripts/oxalis/model/helpers/nml_helpers.ts index 292a9fa4f5..33ac20243a 100644 --- a/frontend/javascripts/oxalis/model/helpers/nml_helpers.ts +++ b/frontend/javascripts/oxalis/model/helpers/nml_helpers.ts @@ -224,6 +224,7 @@ function serializeParameters( name: state.dataset.name, description: annotation.description, organization: state.dataset.owningOrganization, + wkUrl: `${location.protocol}//${location.host}`, }), serializeTag("scale", { x: state.dataset.dataSource.scale[0], From 63ebb85e65559831656a03a3afc82dddc4d0ddbf Mon Sep 17 00:00:00 2001 From: Philipp Otto Date: Wed, 12 Apr 2023 14:55:26 +0200 Subject: [PATCH 07/13] fix that test tries to use global location --- frontend/javascripts/oxalis/model/helpers/nml_helpers.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/frontend/javascripts/oxalis/model/helpers/nml_helpers.ts b/frontend/javascripts/oxalis/model/helpers/nml_helpers.ts index 292a9fa4f5..b9d3183abe 100644 --- a/frontend/javascripts/oxalis/model/helpers/nml_helpers.ts +++ b/frontend/javascripts/oxalis/model/helpers/nml_helpers.ts @@ -27,6 +27,8 @@ import messages from "messages"; import * as Utils from "libs/utils"; import type { BoundingBoxType, Vector3 } from "oxalis/constants"; import Constants from "oxalis/constants"; +import { location } from "libs/window"; + // NML Defaults const DEFAULT_COLOR: Vector3 = [1, 0, 0]; const TASK_BOUNDING_BOX_COLOR: Vector3 = [0, 1, 0]; From 48e89e077012d7a4158e93e89d32df45c90f8e05 Mon Sep 17 00:00:00 2001 From: frcroth Date: Wed, 12 Apr 2023 15:49:28 +0200 Subject: [PATCH 08/13] Fix typos in messages --- conf/messages | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/conf/messages b/conf/messages index cc5737c8c0..e060dbda0c 100644 --- a/conf/messages +++ b/conf/messages @@ -47,7 +47,7 @@ user.notFound=User not found user.noAdmin=Access denied. Only admin users can execute this operation. user.deactivated=Your account has not been activated by an admin yet. Please contact your organization’s admin for help. user.noSelfDeactivate=You cannot deactivate yourself. Please contact an admin to do it for you. -user.lastAdmin=This user is the last remaining admin in your organzation. You cannot remove admin privileges from this account. +user.lastAdmin=This user is the last remaining admin in your organization. You cannot remove admin privileges from this account. user.lastOwner=Cannot deactivate the organization owner. Please talk to the WEBKNOSSOS team to transfer organization ownership. user.email.alreadyInUse=This email address is already in use @@ -68,18 +68,18 @@ oidc.disabled=OIDC is disabled oidc.configuration.invalid=OIDC configuration is invalid braintracing.new=An account on braintracing.org was created for you. You can use the same credentials as on WEBKNOSSOS to login. -braintracing.error=We could not atomatically create an account for you on braintracing.org. Please do it on your own. +braintracing.error=We could not automatically create an account for you on braintracing.org. Please do it on your own. braintracing.exists=Great, you already have an account on braintracing.org. Please double check that you have uploaded all requested information. dataSet=Dataset dataSet.notFound=Dataset {0} does not exist or could not be accessed dataSet.notFoundConsiderLogin=Dataset {0} does not exist or could not be accessed. You may need to log in. dataSet.notFoundForAnnotation=The Dataset for this annotation does not exist or could not be accessed. -dataSet.noAccess=Could not access DataSet {0}. Does your team have access? -dataSet.noAccess.wrongHost=Could not access DataSet {0}. Please check whether you are on the correct WEBKNOSSOS instance. The uploaded file indicates {1} while this instance is {2}. -dataSet.noAccessById=Could not access the corresponding DataSet. This is likely because you are not a member of a team that has access to it. +dataSet.noAccess=Could not access dataset {0}. Does your team have access? +dataSet.noAccess.wrongHost=Could not access dataset {0}. Please check whether you are on the correct WEBKNOSSOS instance. The uploaded file indicates {1} while this instance is {2}. +dataSet.noAccessById=Could not access the corresponding dataset. This is likely because you are not a member of a team that has access to it. dataSet.notImported=Dataset {0} is not imported -dataSet.name.invalid.characters=Dataset name is invalid. Please use only letters, digits, dots, underscores, hypens. +dataSet.name.invalid.characters=Dataset name is invalid. Please use only letters, digits, dots, underscores, hyphens. dataSet.name.invalid.startsWithDot=Dataset name is invalid. Please use a name that does not start with a dot. dataSet.name.invalid.lessThanThreeCharacters=Dataset name is invalid. Please use at least three characters. dataSet.name.alreadyTaken=This name is already being used by a different dataset. Please choose a different name. From 286e738ae00c8244ea03552e7f5b69659ebc3235 Mon Sep 17 00:00:00 2001 From: Philipp Otto Date: Wed, 12 Apr 2023 16:19:09 +0200 Subject: [PATCH 09/13] adapt mocked location to have protocol and host --- frontend/javascripts/libs/window.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/javascripts/libs/window.ts b/frontend/javascripts/libs/window.ts index 68a134086e..b512632601 100644 --- a/frontend/javascripts/libs/window.ts +++ b/frontend/javascripts/libs/window.ts @@ -32,13 +32,13 @@ export const document = const dummyLocation = { ancestorOrigins: [], hash: "", - host: "", + host: "localhost", hostname: "", href: "", origin: "", pathname: "", port: "", - protocol: "", + protocol: "http:", search: "", reload: () => {}, From 8752755d476148758aabdb85ff0c9bb5a3fed5a9 Mon Sep 17 00:00:00 2001 From: frcroth Date: Sun, 16 Apr 2023 12:44:48 +0200 Subject: [PATCH 10/13] Fix test --- test/backend/NMLUnitTestSuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/backend/NMLUnitTestSuite.scala b/test/backend/NMLUnitTestSuite.scala index 637ec97bae..431afed511 100644 --- a/test/backend/NMLUnitTestSuite.scala +++ b/test/backend/NMLUnitTestSuite.scala @@ -45,7 +45,7 @@ class NMLUnitTestSuite extends PlaySpec { } def isParseSuccessful( - parsedTracing: Box[(Option[SkeletonTracing], List[UploadedVolumeLayer], String, String)]): Boolean = + parsedTracing: Box[(Option[SkeletonTracing], List[UploadedVolumeLayer], String, Option[String])]): Boolean = parsedTracing match { case Full(tuple) => tuple match { From d0299c4d630b971b41ef27855015fc649765fe40 Mon Sep 17 00:00:00 2001 From: Philipp Otto Date: Mon, 17 Apr 2023 17:33:18 +0200 Subject: [PATCH 11/13] update nml snapshot --- frontend/javascripts/test/libs/nml.spec.ts | 1 + .../test-bundle/test/libs/nml.spec.js.md | 8 ++++---- .../test-bundle/test/libs/nml.spec.js.snap | Bin 1169 -> 1193 bytes 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/frontend/javascripts/test/libs/nml.spec.ts b/frontend/javascripts/test/libs/nml.spec.ts index 656f14c958..0fc966499f 100644 --- a/frontend/javascripts/test/libs/nml.spec.ts +++ b/frontend/javascripts/test/libs/nml.spec.ts @@ -10,6 +10,7 @@ import EdgeCollection from "oxalis/model/edge_collection"; import { findGroup } from "oxalis/view/right-border-tabs/tree_hierarchy_view_helpers"; import mock from "mock-require"; import test from "ava"; + const TIMESTAMP = 123456789; const buildInfo = { webknossos: { diff --git a/frontend/javascripts/test/snapshots/public-test/test-bundle/test/libs/nml.spec.js.md b/frontend/javascripts/test/snapshots/public-test/test-bundle/test/libs/nml.spec.js.md index ca81a97bb7..0f46424d07 100644 --- a/frontend/javascripts/test/snapshots/public-test/test-bundle/test/libs/nml.spec.js.md +++ b/frontend/javascripts/test/snapshots/public-test/test-bundle/test/libs/nml.spec.js.md @@ -1,8 +1,8 @@ -# Snapshot report for `public/test-bundle/test/libs/nml.spec.js` +# Snapshot report for `public-test/test-bundle/test/libs/nml.spec.js` The actual snapshot is saved in `nml.spec.js.snap`. -Generated by [AVA](https://ava.li). +Generated by [AVA](https://avajs.dev). ## nml @@ -14,7 +14,7 @@ Generated by [AVA](https://ava.li). ␊ ␊ ␊ - ␊ + ␊ ␊ ␊