-
Notifications
You must be signed in to change notification settings - Fork 24
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* add N5 layer classes * first working version * refactor zarr and n5 with common superclasses, and refactor datareader in simpler composition instead of inheritance * add PaddedChunkReader * rename zarrMagOpt to n5magopt * add CompressionOption * update changelog * Update webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/n5/ChunkReaderN5.scala Co-authored-by: Florian M <[email protected]> * incorporate pr feedback * reformat * Fix JsonImplicits import, rename DatasetLocatorMag to MagLocator Co-authored-by: leowe <[email protected]> Co-authored-by: Florian M <[email protected]> Co-authored-by: Florian M <[email protected]>
- Loading branch information
4 people
authored
Sep 22, 2022
1 parent
0f57cc3
commit 6d688e9
Showing
44 changed files
with
1,208 additions
and
681 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
30 changes: 30 additions & 0 deletions
30
webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/MagLocator.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,30 @@ | ||
| package com.scalableminds.webknossos.datastore.dataformats | ||
|
|
||
| import com.scalableminds.util.geometry.Vec3Int | ||
| import com.scalableminds.webknossos.datastore.dataformats.zarr.{FileSystemCredentials, RemoteSourceDescriptor} | ||
| import com.scalableminds.webknossos.datastore.datareaders.AxisOrder | ||
| import com.scalableminds.webknossos.datastore.models.datasource.ResolutionFormatHelper | ||
| import com.scalableminds.webknossos.datastore.storage.FileSystemsHolder | ||
| import play.api.libs.json.{Json, OFormat} | ||
|
|
||
| import java.net.URI | ||
|
|
||
| case class MagLocator(mag: Vec3Int, | ||
| path: Option[String], | ||
| credentials: Option[FileSystemCredentials], | ||
| axisOrder: Option[AxisOrder]) { | ||
|
|
||
| lazy val pathWithFallback: String = path.getOrElse(mag.toMagLiteral(allowScalar = true)) | ||
| private lazy val uri: URI = new URI(pathWithFallback) | ||
| private lazy val isRemote: Boolean = FileSystemsHolder.isSupportedRemoteScheme(uri.getScheme) | ||
| lazy val remoteSource: Option[RemoteSourceDescriptor] = | ||
| if (isRemote) | ||
| Some(RemoteSourceDescriptor(uri, credentials.map(_.user), credentials.flatMap(_.password))) | ||
| else | ||
| None | ||
|
|
||
| } | ||
|
|
||
| object MagLocator extends ResolutionFormatHelper { | ||
| implicit val jsonFormat: OFormat[MagLocator] = Json.format[MagLocator] | ||
| } |
54 changes: 54 additions & 0 deletions
54
...atastore/app/com/scalableminds/webknossos/datastore/dataformats/n5/N5BucketProvider.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,54 @@ | ||
| package com.scalableminds.webknossos.datastore.dataformats.n5 | ||
|
|
||
| import com.scalableminds.util.geometry.Vec3Int | ||
| import com.scalableminds.util.requestlogging.RateLimitedErrorLogging | ||
| import com.scalableminds.util.tools.Fox | ||
| import com.scalableminds.webknossos.datastore.dataformats.{BucketProvider, DataCubeHandle, MagLocator} | ||
| import com.scalableminds.webknossos.datastore.models.BucketPosition | ||
| import com.scalableminds.webknossos.datastore.models.requests.DataReadInstruction | ||
| import com.scalableminds.webknossos.datastore.datareaders.n5.N5Array | ||
| import com.typesafe.scalalogging.LazyLogging | ||
| import net.liftweb.common.{Box, Empty, Failure, Full} | ||
| import net.liftweb.util.Helpers.tryo | ||
|
|
||
| import java.nio.file.Path | ||
| import scala.concurrent.ExecutionContext | ||
|
|
||
| class N5CubeHandle(n5Array: N5Array) extends DataCubeHandle with LazyLogging with RateLimitedErrorLogging { | ||
|
|
||
| def cutOutBucket(bucket: BucketPosition)(implicit ec: ExecutionContext): Fox[Array[Byte]] = { | ||
| val shape = Vec3Int.full(bucket.bucketLength) | ||
| val offset = Vec3Int(bucket.voxelXInMag, bucket.voxelYInMag, bucket.voxelZInMag) | ||
| n5Array.readBytesXYZ(shape, offset).recover { | ||
| case t: Throwable => logError(t); Failure(t.getMessage, Full(t), Empty) | ||
| } | ||
| } | ||
|
|
||
| override protected def onFinalize(): Unit = () | ||
|
|
||
| } | ||
|
|
||
| class N5BucketProvider(layer: N5Layer) extends BucketProvider with LazyLogging with RateLimitedErrorLogging { | ||
|
|
||
| override def loadFromUnderlying(readInstruction: DataReadInstruction): Box[N5CubeHandle] = { | ||
| val n5MagOpt: Option[MagLocator] = | ||
| layer.mags.find(_.mag == readInstruction.bucket.mag) | ||
|
|
||
| n5MagOpt match { | ||
| case None => Empty | ||
| case Some(n5Mag) => | ||
| val magPathOpt: Option[Path] = { | ||
| n5Mag.remoteSource match { | ||
| case Some(remoteSource) => remotePathFrom(remoteSource) | ||
| case None => localPathFrom(readInstruction, n5Mag.pathWithFallback) | ||
| } | ||
| } | ||
| magPathOpt match { | ||
| case None => Empty | ||
| case Some(magPath) => | ||
| tryo(onError = e => logError(e))(N5Array.open(magPath, n5Mag.axisOrder)).map(new N5CubeHandle(_)) | ||
| } | ||
| } | ||
|
|
||
| } | ||
| } |
55 changes: 55 additions & 0 deletions
55
...os-datastore/app/com/scalableminds/webknossos/datastore/dataformats/n5/N5DataLayers.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,55 @@ | ||
| package com.scalableminds.webknossos.datastore.dataformats.n5 | ||
|
|
||
| import com.scalableminds.util.geometry.{BoundingBox, Vec3Int} | ||
| import com.scalableminds.webknossos.datastore.dataformats.MagLocator | ||
| import com.scalableminds.webknossos.datastore.models.datasource.LayerViewConfiguration.LayerViewConfiguration | ||
| import com.scalableminds.webknossos.datastore.models.datasource._ | ||
| import play.api.libs.json.{Json, OFormat} | ||
|
|
||
| trait N5Layer extends DataLayer { | ||
|
|
||
| val dataFormat: DataFormat.Value = DataFormat.n5 | ||
|
|
||
| lazy val bucketProvider = new N5BucketProvider(this) | ||
|
|
||
| def resolutions: List[Vec3Int] = mags.map(_.mag) | ||
|
|
||
| def mags: List[MagLocator] | ||
|
|
||
| def lengthOfUnderlyingCubes(resolution: Vec3Int): Int = Int.MaxValue // Prevents the wkw-shard-specific handle caching | ||
|
|
||
| def numChannels: Option[Int] = Some(if (elementClass == ElementClass.uint24) 3 else 1) | ||
|
|
||
| } | ||
|
|
||
| case class N5DataLayer( | ||
| name: String, | ||
| category: Category.Value, | ||
| boundingBox: BoundingBox, | ||
| elementClass: ElementClass.Value, | ||
| mags: List[MagLocator], | ||
| defaultViewConfiguration: Option[LayerViewConfiguration] = None, | ||
| adminViewConfiguration: Option[LayerViewConfiguration] = None, | ||
| override val numChannels: Option[Int] = Some(1) | ||
| ) extends N5Layer | ||
|
|
||
| object N5DataLayer { | ||
| implicit val jsonFormat: OFormat[N5DataLayer] = Json.format[N5DataLayer] | ||
| } | ||
|
|
||
| case class N5SegmentationLayer( | ||
| name: String, | ||
| boundingBox: BoundingBox, | ||
| elementClass: ElementClass.Value, | ||
| mags: List[MagLocator], | ||
| largestSegmentId: Long, | ||
| mappings: Option[Set[String]] = None, | ||
| defaultViewConfiguration: Option[LayerViewConfiguration] = None, | ||
| adminViewConfiguration: Option[LayerViewConfiguration] = None, | ||
| override val numChannels: Option[Int] = Some(1) | ||
| ) extends SegmentationLayer | ||
| with N5Layer | ||
|
|
||
| object N5SegmentationLayer { | ||
| implicit val jsonFormat: OFormat[N5SegmentationLayer] = Json.format[N5SegmentationLayer] | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.