Merge branch 'master' of github.com:scalableminds/webknossos into red…

…esign-right-sidebar

* 'master' of github.com:scalableminds/webknossos:
  Create bounding box by dragging with box tool (#7118)
  Prevent 'negative' buckets from being created (#7124)
  Lazy load onnx and canvas2html module (#7121)
  Disable editing of super voxel skeletons in skeleton mode (#7086)
  add missing evolution to migration guide (#7126)
  Change sttp backend to HttpURLConnectionBackend (#7125)
  Implement Zarr v3 and sharding codec (#7079)
  Fix decompression of garbage data after valid gzip data causing decompression to fail (#7119)
  When scanning volume buckets, skip those with unparseable key (#7115)

CHANGELOG.unreleased.md

@@ -12,11 +12,18 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released
 
 ### Added
 - Subfolders of the currently active folder are now also rendered in the dataset table in the dashboard. [#6996](https://github.com/scalableminds/webknossos/pull/6996)
+- Add ability to view [zarr v3](https://zarr-specs.readthedocs.io/en/latest/v3/core/v3.0.html) datasets. [#7079](https://github.com/scalableminds/webknossos/pull/7079)
 
 ### Changed
+- Creating bounding boxes can now be done by dragging the left mouse button (when the bounding box tool is selected). To move around in the dataset while this tool is active, keep ALT pressed. [#7118](https://github.com/scalableminds/webknossos/pull/7118)
+- Agglomerate skeletons can only be modified if the proofreading tool is active so they stay in sync with the underlying segmentation and agglomerate graph. Agglomerate skeletons cannot be modified using any other means. They are marked in the skeleton list using the clipboard icon of the proofreading tool. When exporting skeletons in the NML format, trees ("things") now have a `type` property which is either "DEFAULT" or "AGGLOMERATE". [#7086](https://github.com/scalableminds/webknossos/pull/7086)
 - Redesigned the info tab in the right-hand sidebar to be fit the new branding and design language. [#7110](https://github.com/scalableminds/webknossos/pull/7110)
 
 ### Fixed
+- Fixed a bug where some volume annotations could not be downloaded. [#7115](https://github.com/scalableminds/webknossos/pull/7115)
+- Fixed reading of some remote datasets where invalid data would follow valid gzip data, causing the decompression to fail. [#7119](https://github.com/scalableminds/webknossos/pull/7119)
+- Fixed problems which could arise when annotating volume data at negative positions (which is not supported and is properly ignored now). [#7124](https://github.com/scalableminds/webknossos/pull/7124)
+- Fixed some requests failing for streaming remote data via HTTP, which was observed when streaming data via Zarr from another WEBKNOSSOS instance. [#7125](https://github.com/scalableminds/webknossos/pull/7125)
 
 ### Removed
 - Support for [webknososs-connect](https://github.com/scalableminds/webknossos-connect) data store servers has been removed. Use the "Add Remote Dataset" functionality instead. [#7031](https://github.com/scalableminds/webknossos/pull/7031)

MIGRATIONS.unreleased.md

@@ -9,3 +9,4 @@ User-facing changes are documented in the [changelog](CHANGELOG.released.md).
 - Support for [webknososs-connect](https://github.com/scalableminds/webknossos-connect) data store servers has been removed. Please remove the database entries for such datastores and the corresponding datasets and annotations. If you need to keep the datasets, consider adding them to a regular datastore using the same name. If the webknossos datastore does not support the dataset format, it may make sense to manually move the datasets to an existing datastore in the database, to avoid breaking foreign key relations. They will then be shown as “no longer available on the datastore”. [#7032](https://github.com/scalableminds/webknossos/pull/7032)
 
 ### Postgres Evolutions:
+- [102-no-more-wkconnect.sql](conf/evolutions/102-no-more-wkconnect.sql)

app/models/annotation/nml/NmlParser.scala

@@ -8,6 +8,7 @@ import com.scalableminds.webknossos.datastore.geometry.{ColorProto, NamedBoundin
 import com.scalableminds.webknossos.datastore.helpers.{NodeDefaults, ProtoGeometryImplicits, SkeletonTracingDefaults}
 import com.scalableminds.webknossos.datastore.models.datasource.ElementClass
 import com.scalableminds.webknossos.tracingstore.tracings.ColorGenerator
+import com.scalableminds.webknossos.tracingstore.tracings.skeleton.updating.TreeType
 import com.scalableminds.webknossos.tracingstore.tracings.skeleton.{MultiComponentTreeSplitter, TreeValidator}
 import com.typesafe.scalalogging.LazyLogging
 import models.annotation.UploadedVolumeLayer
@@ -162,7 +163,7 @@ object NmlParser extends LazyLogging with ProtoGeometryImplicits with ColorGener
       }
     )
 
-  def extractSegmentGroups(segmentGroupContainerNodes: NodeSeq)(
+  private def extractSegmentGroups(segmentGroupContainerNodes: NodeSeq)(
       implicit m: MessagesProvider): Box[List[SegmentGroup]] = {
     val segmentGroupNodes = segmentGroupContainerNodes.flatMap(_ \ "group")
     segmentGroupNodes.map(parseSegmentGroup).toList.toSingleBox(Messages("nml.element.invalid", "segment groups"))
@@ -321,6 +322,13 @@ object NmlParser extends LazyLogging with ProtoGeometryImplicits with ColorGener
   private def parseName(node: XMLNode) =
     getSingleAttribute(node, "name")
 
+  private def parseType(node: XMLNode): Option[TreeTypeProto] =
+    for {
+      asString <- getSingleAttributeOpt(node, "type")
+      asScalaEnum <- TreeType.fromString(asString)
+      asProtoEnum = TreeType.toProto(asScalaEnum)
+    } yield asProtoEnum
+
   private def parseGroupId(node: XMLNode) =
     getSingleAttribute(node, "groupId").toIntOpt
 
@@ -337,6 +345,7 @@ object NmlParser extends LazyLogging with ProtoGeometryImplicits with ColorGener
       id <- treeIdText.toIntOpt ?~ Messages("nml.tree.id.invalid", treeIdText)
       color = parseColorOpt(tree)
       name = parseName(tree)
+      treeType = parseType(tree)
       groupId = parseGroupId(tree)
       isVisible = parseVisibility(tree, color)
       nodes <- (tree \ "nodes" \ "node")
@@ -352,7 +361,18 @@ object NmlParser extends LazyLogging with ProtoGeometryImplicits with ColorGener
       treeComments = nodeIds.flatMap(nodeId => comments.getOrElse(nodeId, List()))
       createdTimestamp = if (nodes.isEmpty) System.currentTimeMillis()
       else nodes.minBy(_.createdTimestamp).createdTimestamp
-    } yield Tree(id, nodes, edges, color, treeBranchPoints, treeComments, name, createdTimestamp, groupId, isVisible)
+    } yield
+      Tree(id,
+           nodes,
+           edges,
+           color,
+           treeBranchPoints,
+           treeComments,
+           name,
+           createdTimestamp,
+           groupId,
+           isVisible,
+           treeType)
   }
 
   private def parseComments(comments: NodeSeq)(implicit m: MessagesProvider): Box[List[Comment]] =

app/models/annotation/nml/NmlWriter.scala

@@ -271,6 +271,7 @@ class NmlWriter @Inject()(implicit ec: ExecutionContext) extends FoxImplicits {
         writeColor(t.color)
         writer.writeAttribute("name", t.name)
         t.groupId.foreach(groupId => writer.writeAttribute("groupId", groupId.toString))
+        t.`type`.foreach(t => writer.writeAttribute("type", t.toString))
         Xml.withinElementSync("nodes")(writeNodesAsXml(t.nodes.sortBy(_.id)))
         Xml.withinElementSync("edges")(writeEdgesAsXml(t.edges))
       }

app/models/binary/explore/ExploreRemoteLayerService.scala

@@ -7,9 +7,12 @@ import com.scalableminds.webknossos.datastore.dataformats.precomputed.{
   PrecomputedDataLayer,
   PrecomputedSegmentationLayer
 }
+import com.scalableminds.webknossos.datastore.dataformats.zarr3.{Zarr3DataLayer, Zarr3SegmentationLayer}
 import com.scalableminds.webknossos.datastore.dataformats.zarr._
 import com.scalableminds.webknossos.datastore.datareaders.n5.N5Header
+import com.scalableminds.webknossos.datastore.datareaders.precomputed.PrecomputedHeader
 import com.scalableminds.webknossos.datastore.datareaders.zarr._
+import com.scalableminds.webknossos.datastore.datareaders.zarr3.Zarr3ArrayHeader
 import com.scalableminds.webknossos.datastore.datavault.VaultPath
 import com.scalableminds.webknossos.datastore.models.datasource._
 import com.scalableminds.webknossos.datastore.storage.{DataVaultsHolder, RemoteSourceDescriptor}
@@ -142,6 +145,12 @@ class ExploreRemoteLayerService @Inject()(credentialService: CredentialService)
           case l: PrecomputedSegmentationLayer =>
             l.copy(mags = l.mags.map(mag => mag.copy(mag = mag.mag * magFactors)),
                    boundingBox = l.boundingBox * magFactors)
+          case l: Zarr3DataLayer =>
+            l.copy(mags = l.mags.map(mag => mag.copy(mag = mag.mag * magFactors)),
+                   boundingBox = l.boundingBox * magFactors)
+          case l: Zarr3SegmentationLayer =>
+            l.copy(mags = l.mags.map(mag => mag.copy(mag = mag.mag * magFactors)),
+                   boundingBox = l.boundingBox * magFactors)
           case _ => throw new Exception("Encountered unsupported layer format during explore remote")
         }
       })
@@ -168,12 +177,15 @@ class ExploreRemoteLayerService @Inject()(credentialService: CredentialService)
         remotePath,
         credentialId.map(_.toString),
         reportMutable,
-        List(new ZarrArrayExplorer,
-             new NgffExplorer,
-             new WebknossosZarrExplorer,
-             new N5ArrayExplorer,
-             new N5MultiscalesExplorer,
-             new PrecomputedExplorer)
+        List(
+          new ZarrArrayExplorer,
+          new NgffExplorer,
+          new WebknossosZarrExplorer,
+          new N5ArrayExplorer,
+          new N5MultiscalesExplorer,
+          new PrecomputedExplorer,
+          new Zarr3ArrayExplorer
+        )
       )
     } yield layersWithVoxelSizes
 
@@ -183,6 +195,8 @@ class ExploreRemoteLayerService @Inject()(credentialService: CredentialService)
     else if (uri.endsWith(NgffMetadata.FILENAME_DOT_ZATTRS)) uri.dropRight(NgffMetadata.FILENAME_DOT_ZATTRS.length)
     else if (uri.endsWith(NgffGroupHeader.FILENAME_DOT_ZGROUP))
       uri.dropRight(NgffGroupHeader.FILENAME_DOT_ZGROUP.length)
+    else if (uri.endsWith(PrecomputedHeader.FILENAME_INFO)) uri.dropRight(PrecomputedHeader.FILENAME_INFO.length)
+    else if (uri.endsWith(Zarr3ArrayHeader.ZARR_JSON)) uri.dropRight(Zarr3ArrayHeader.ZARR_JSON.length)
     else uri
 
   private def exploreRemoteLayersForRemotePath(

app/models/binary/explore/Zarr3ArrayExplorer.scala

@@ -0,0 +1,38 @@
+package models.binary.explore
+
+import com.scalableminds.util.geometry.{Vec3Double, Vec3Int}
+import com.scalableminds.util.tools.Fox
+import com.scalableminds.webknossos.datastore.dataformats.MagLocator
+import com.scalableminds.webknossos.datastore.dataformats.zarr3.{Zarr3DataLayer, Zarr3Layer, Zarr3SegmentationLayer}
+import com.scalableminds.webknossos.datastore.datareaders.AxisOrder
+import com.scalableminds.webknossos.datastore.datareaders.zarr3.Zarr3ArrayHeader
+import com.scalableminds.webknossos.datastore.datavault.VaultPath
+import com.scalableminds.webknossos.datastore.models.datasource.Category
+
+import scala.concurrent.ExecutionContext.Implicits.global
+
+class Zarr3ArrayExplorer extends RemoteLayerExplorer {
+
+  override def name: String = "Zarr v3 Array"
+
+  override def explore(remotePath: VaultPath, credentialId: Option[String]): Fox[List[(Zarr3Layer, Vec3Double)]] =
+    for {
+      zarrayPath <- Fox.successful(remotePath / Zarr3ArrayHeader.ZARR_JSON)
+      name = guessNameFromPath(remotePath)
+      zarrHeader <- parseJsonFromPath[Zarr3ArrayHeader](zarrayPath) ?~> s"failed to read zarr v3 header at $zarrayPath"
+      _ <- zarrHeader.assertValid
+      elementClass <- zarrHeader.elementClass ?~> "failed to read element class from zarr header"
+      guessedAxisOrder = AxisOrder.asCxyzFromRank(zarrHeader.rank)
+      boundingBox <- zarrHeader.boundingBox(guessedAxisOrder) ?~> "failed to read bounding box from zarr header. Make sure data is in (T/C)ZYX format"
+      magLocator = MagLocator(Vec3Int.ones,
+                              Some(remotePath.toUri.toString),
+                              None,
+                              Some(guessedAxisOrder),
+                              None,
+                              credentialId)
+      layer: Zarr3Layer = if (looksLikeSegmentationLayer(name, elementClass)) {
+        Zarr3SegmentationLayer(name, boundingBox, elementClass, List(magLocator), largestSegmentId = None)
+      } else Zarr3DataLayer(name, Category.color, boundingBox, elementClass, List(magLocator))
+    } yield List((layer, Vec3Double(1.0, 1.0, 1.0)))
+
+}

frontend/javascripts/admin/task/task_create_form_view.tsx

@@ -782,7 +782,9 @@ class TaskCreateFormView extends React.PureComponent<Props, State> {
                   value={this.state.specificationType}
                   onChange={(evt: RadioChangeEvent) =>
                     this.setState({
-                      specificationType: coalesce(SpecificationEnum, evt.target.value),
+                      specificationType:
+                        coalesce(SpecificationEnum, evt.target.value) ||
+                        this.state.specificationType,
                     })
                   }
                 >

frontend/javascripts/dashboard/advanced_dataset/create_explorative_modal.tsx

@@ -262,7 +262,7 @@ function CreateExplorativeModal({ datasetId, onClose }: Props) {
   return (
     <Modal
       title={`Create New Annotation for Dataset “${datasetId.name}”`}
-      visible
+      open
       width={500}
       footer={null}
       onCancel={onClose}

frontend/javascripts/libs/utils.ts

@@ -1113,14 +1113,8 @@ export function diffObjects(
   return changes(object, base);
 }
 
-export function coalesce<T extends string | number | symbol>(
-  obj: { [key: string]: T },
-  field: T,
-): T | null {
-  if (obj && typeof obj === "object" && (field in obj || Object.values(obj).includes(field))) {
-    return field;
-  }
-  return null;
+export function coalesce<T extends {}>(e: T, token: any): T[keyof T] | null {
+  return Object.values(e).includes(token as T[keyof T]) ? token : null;
 }
 
 export function pluralize(str: string, count: number, optPluralForm: string | null = null): string {

frontend/javascripts/messages.tsx

@@ -405,6 +405,8 @@ instead. Only enable this option if you understand its effect. All layers will n
   "nml.expected_attribute_missing":
     "Attribute with the following name was expected, but is missing or empty:",
   "nml.invalid_timestamp": "Attribute with the following name was expected to be a unix timestamp:",
+  "nml.invalid_tree_type":
+    "Attribute with the following name was expected to be a valid tree type:",
   "nml.branchpoint_without_tree":
     "NML contains <branchpoint ...> with a node id that is not in any tree: Node with id",
   "nml.comment_without_tree":

frontend/javascripts/oxalis/api/api_latest.ts

@@ -73,7 +73,7 @@ import {
 } from "oxalis/model/accessors/volumetracing_accessor";
 import { getHalfViewportExtentsFromState } from "oxalis/model/sagas/saga_selectors";
 import {
-  getLayerBoundaries,
+  getLayerBoundingBox,
   getLayerByName,
   getResolutionInfo,
   getVisibleSegmentationLayer,
@@ -1363,11 +1363,8 @@ class DataApi {
    * Returns the bounding box for a given layer name.
    */
   getBoundingBox(layerName: string): [Vector3, Vector3] {
-    const { lowerBoundary, upperBoundary } = getLayerBoundaries(
-      Store.getState().dataset,
-      layerName,
-    );
-    return [lowerBoundary, upperBoundary];
+    const { min, max } = getLayerBoundingBox(Store.getState().dataset, layerName);
+    return [min, max];
   }
 
   /**

frontend/javascripts/oxalis/api/api_v2.ts

@@ -22,7 +22,6 @@ import {
   getActiveTree,
   getTree,
 } from "oxalis/model/accessors/skeletontracing_accessor";
-import { getLayerBoundaries } from "oxalis/model/accessors/dataset_accessor";
 import { setActiveCellAction } from "oxalis/model/actions/volumetracing_actions";
 import { getActiveCellId } from "oxalis/model/accessors/volumetracing_accessor";
 import type { Vector3, AnnotationTool, ControlMode } from "oxalis/constants";
@@ -57,6 +56,7 @@ import { APICompoundType, APICompoundTypeEnum } from "types/api_flow_types";
 import { coalesce } from "libs/utils";
 
 import { assertExists, assertSkeleton, assertVolume } from "./api_latest";
+import { getLayerBoundingBox } from "oxalis/model/accessors/dataset_accessor";
 
 function makeTreeBackwardsCompatible(tree: TreeMap) {
   return update(tree, {
@@ -569,15 +569,12 @@ class DataApi {
 
   /**
    * Returns the bounding box for a given layer name. Note that the described interval
-     is half-open, meaning that the lowerBoundary is included and the upperBoundary is not
+     is half-open, meaning that the lower boundary is included and the upper boundary is not
      included in the bounding box.
    */
   getBoundingBox(layerName: string): [Vector3, Vector3] {
-    const { lowerBoundary, upperBoundary } = getLayerBoundaries(
-      Store.getState().dataset,
-      layerName,
-    );
-    return [lowerBoundary, upperBoundary];
+    const { min, max } = getLayerBoundingBox(Store.getState().dataset, layerName);
+    return [min, max];
   }
 
   /**

frontend/javascripts/oxalis/constants.ts

@@ -241,6 +241,11 @@ export const enum MappingStatusEnum {
   ENABLED = "ENABLED",
 }
 export type MappingStatus = keyof typeof MappingStatusEnum;
+export enum TreeTypeEnum {
+  DEFAULT = "DEFAULT",
+  AGGLOMERATE = "AGGLOMERATE",
+}
+export type TreeType = keyof typeof TreeTypeEnum;
 export const NODE_ID_REF_REGEX = /#([0-9]+)/g;
 export const POSITION_REF_REGEX = /#\(([0-9]+,[0-9]+,[0-9]+)\)/g;
 // The plane in orthogonal mode is a little smaller than the viewport