diff --git a/CHANGELOG.unreleased.md b/CHANGELOG.unreleased.md index ffdd320c19..a2c8e22ef8 100644 --- a/CHANGELOG.unreleased.md +++ b/CHANGELOG.unreleased.md @@ -27,12 +27,13 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released - Migrated nightly screenshot tests from CircleCI to GitHub actions. [#8134](https://github.com/scalableminds/webknossos/pull/8134) - Migrated nightly screenshot tests for wk.org from CircleCI to GitHub actions. [#8135](https://github.com/scalableminds/webknossos/pull/8135) - Thumbnails for datasets now use the selected mapping from the view configuration if available. [#8157](https://github.com/scalableminds/webknossos/pull/8157) +- Reading image files on datastore filesystem is now done asynchronously. [#8126](https://github.com/scalableminds/webknossos/pull/8126) ### Fixed - Fixed a bug during dataset upload in case the configured `datastore.baseFolder` is an absolute path. [#8098](https://github.com/scalableminds/webknossos/pull/8098) [#8103](https://github.com/scalableminds/webknossos/pull/8103) - Fixed bbox export menu item [#8152](https://github.com/scalableminds/webknossos/pull/8152) - When trying to save an annotation opened via a link including a sharing token, the token is automatically discarded in case it is insufficient for update actions but the users token is. [#8139](https://github.com/scalableminds/webknossos/pull/8139) -- Fix that scrolling in the trees and segments tab did not work while dragging. [#8162](https://github.com/scalableminds/webknossos/pull/8162) +- Fix that scrolling in the trees and segments tab did not work while dragging. [#8162](https://github.com/scalableminds/webknossos/pull/8162) - Fixed that uploading a dataset which needs a conversion failed when the angstrom unit was configured for the conversion. [#8173](https://github.com/scalableminds/webknossos/pull/8173) - Fixed that the skeleton search did not automatically expand groups that contained the selected tree [#8129](https://github.com/scalableminds/webknossos/pull/8129) - Fixed interactions in the trees and segments tab like the search due to a bug introduced by [#8162](https://github.com/scalableminds/webknossos/pull/8162). [#8186](https://github.com/scalableminds/webknossos/pull/8186) diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datavault/FileSystemDataVault.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datavault/FileSystemDataVault.scala index bb16d34cd0..0d42244c6a 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datavault/FileSystemDataVault.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datavault/FileSystemDataVault.scala @@ -1,15 +1,16 @@ package com.scalableminds.webknossos.datastore.datavault import com.scalableminds.util.tools.Fox -import net.liftweb.common.Box.tryo -import com.scalableminds.util.tools.Fox.{bool2Fox, box2Fox} +import com.scalableminds.util.tools.Fox.bool2Fox import com.scalableminds.webknossos.datastore.storage.DataVaultService +import net.liftweb.common.{Box, Full} import org.apache.commons.lang3.builder.HashCodeBuilder import java.nio.ByteBuffer -import java.nio.file.{Files, Path, Paths} +import java.nio.channels.{AsynchronousFileChannel, CompletionHandler} +import java.nio.file.{Files, Path, Paths, StandardOpenOption} import java.util.stream.Collectors -import scala.concurrent.ExecutionContext +import scala.concurrent.{ExecutionContext, Promise} import scala.jdk.CollectionConverters._ class FileSystemDataVault extends DataVault { @@ -24,31 +25,55 @@ class FileSystemDataVault extends DataVault { private def readBytesLocal(localPath: Path, range: RangeSpecifier)(implicit ec: ExecutionContext): Fox[Array[Byte]] = if (Files.exists(localPath)) { range match { - case Complete() => tryo(Files.readAllBytes(localPath)).toFox + case Complete() => + readAsync(localPath, 0, Math.toIntExact(Files.size(localPath))) + case StartEnd(r) => - tryo { - val channel = Files.newByteChannel(localPath) - val buf = ByteBuffer.allocateDirect(r.length) - channel.position(r.start) - channel.read(buf) - buf.rewind() - val arr = new Array[Byte](r.length) - buf.get(arr) - arr - }.toFox + readAsync(localPath, r.start, r.length) + case SuffixLength(length) => - tryo { - val channel = Files.newByteChannel(localPath) - val buf = ByteBuffer.allocateDirect(length) - channel.position(channel.size() - length) - channel.read(buf) - buf.rewind() - val arr = new Array[Byte](length) - buf.get(arr) - arr - }.toFox + val fileSize = Files.size(localPath) + readAsync(localPath, fileSize - length, length) } - } else Fox.empty + } else { + Fox.empty + } + + private def readAsync(path: Path, position: Long, length: Int)(implicit ec: ExecutionContext): Fox[Array[Byte]] = { + val promise = Promise[Box[Array[Byte]]]() + val buffer = ByteBuffer.allocateDirect(length) + var channel: AsynchronousFileChannel = null + + try { + channel = AsynchronousFileChannel.open(path, StandardOpenOption.READ) + + channel.read( + buffer, + position, + buffer, + new CompletionHandler[Integer, ByteBuffer] { + override def completed(result: Integer, buffer: ByteBuffer): Unit = { + buffer.rewind() + val arr = new Array[Byte](length) + buffer.get(arr) + promise.success(Full(arr)) + channel.close() + } + + override def failed(exc: Throwable, buffer: ByteBuffer): Unit = { + promise.failure(exc) + channel.close() + } + } + ) + } catch { + case e: Throwable => + promise.failure(e) + if (channel != null && channel.isOpen) channel.close() + } + + promise.future + } override def listDirectory(path: VaultPath, maxItems: Int)(implicit ec: ExecutionContext): Fox[List[VaultPath]] = vaultPathToLocalPath(path).map(