Explore remote datasets in datastore (#7800)

* WIP: explore remote datasets in datastore * WIP: move ExploreRemoteLayerService to datastore * move more stuff * move more stuff * start to connect the pieces * look up credentials, cleanup * allow selecting datastore in frontend * changelog * implement pr feedback
scalableminds · May 22, 2024 · e443ee5 · e443ee5
1 parent 93258c2
commit e443ee5
Show file tree

Hide file tree

Showing 18 changed files with 370 additions and 236 deletions.
diff --git a/CHANGELOG.unreleased.md b/CHANGELOG.unreleased.md
@@ -21,6 +21,7 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released
 - Non-admin or -manager users can no longer start long-running jobs that create datasets. This includes annotation materialization and AI inferrals. [#7753](https://github.com/scalableminds/webknossos/pull/7753)
 - In the time tracking view, all annotations and tasks can be shown for each user by expanding the table. The individual time spans spent with a task or annotating an explorative annotation can be accessed via CSV export. The detail view including a chart for the individual spans has been removed. [#7733](https://github.com/scalableminds/webknossos/pull/7733)
 - Slightly refactored the `<FixExpandleTable/>`component to use columns as props. [#7772](https://github.com/scalableminds/webknossos/pull/7772)
+- The config value `datastore.localFolderWhitelist` can now be set for each datastore individually. [#7800](https://github.com/scalableminds/webknossos/pull/7800)
 
 ### Fixed
 - Fixed a bug where a toast that was reopened had a flickering effect during the reopening animation. [#7793](https://github.com/scalableminds/webknossos/pull/7793)

diff --git a/app/controllers/AnnotationIOController.scala b/app/controllers/AnnotationIOController.scala
@@ -1,5 +1,7 @@
 package controllers
 
+import collections.SequenceUtils
+
 import java.io.{BufferedOutputStream, File, FileOutputStream}
 import java.util.zip.Deflater
 import org.apache.pekko.actor.ActorSystem
@@ -266,11 +268,7 @@ class AnnotationIOController @Inject()(
     }
 
   private def assertAllOnSameDataset(skeletons: List[SkeletonTracing], volumes: List[VolumeTracing]): Fox[String] =
-    for {
-      datasetName <- volumes.headOption.map(_.datasetName).orElse(skeletons.headOption.map(_.datasetName)).toFox
-      _ <- bool2Fox(skeletons.forall(_.datasetName == datasetName))
-      _ <- bool2Fox(volumes.forall(_.datasetName == datasetName))
-    } yield datasetName
+    SequenceUtils.findUniqueElement(volumes.map(_.datasetName) ++ skeletons.map(_.datasetName)).toFox
 
   private def assertAllOnSameOrganization(skeletons: List[SkeletonTracing],
                                           volumes: List[VolumeTracing]): Fox[Option[String]] = {

diff --git a/app/controllers/DatasetController.scala b/app/controllers/DatasetController.scala
@@ -5,26 +5,24 @@ import com.scalableminds.util.accesscontext.{DBAccessContext, GlobalAccessContex
 import com.scalableminds.util.geometry.{BoundingBox, Vec3Int}
 import com.scalableminds.util.time.Instant
 import com.scalableminds.util.tools.{Fox, TristateOptionJsonHelper}
-import com.scalableminds.webknossos.datastore.models.datasource.{DataLayer, ElementClass, GenericDataSource}
+import com.scalableminds.webknossos.datastore.models.datasource.ElementClass
 import models.analytics.{AnalyticsService, ChangeDatasetSettingsEvent, OpenDatasetEvent}
 import models.dataset._
 import models.dataset.explore.{
   ExploreAndAddRemoteDatasetParameters,
-  ExploreRemoteDatasetParameters,
-  ExploreRemoteLayerService
+  WKExploreRemoteLayerParameters,
+  WKExploreRemoteLayerService
 }
 import models.organization.OrganizationDAO
 import models.team.{TeamDAO, TeamService}
 import models.user.{User, UserDAO, UserService}
-import net.liftweb.common.{Box, Empty, Failure, Full}
 import play.api.i18n.{Messages, MessagesProvider}
 import play.api.libs.functional.syntax._
 import play.api.libs.json._
 import play.api.mvc.{Action, AnyContent, PlayBodyParsers}
 import utils.{ObjectId, WkConf}
 
 import javax.inject.Inject
-import scala.collection.mutable.ListBuffer
 import scala.concurrent.{ExecutionContext, Future}
 import com.scalableminds.webknossos.datastore.models.AdditionalCoordinate
 import mail.{MailchimpClient, MailchimpTag}
@@ -71,7 +69,7 @@ class DatasetController @Inject()(userService: UserService,
                                   conf: WkConf,
                                   analyticsService: AnalyticsService,
                                   mailchimpClient: MailchimpClient,
-                                  exploreRemoteLayerService: ExploreRemoteLayerService,
+                                  wkExploreRemoteLayerService: WKExploreRemoteLayerService,
                                   sil: Silhouette[WkEnv])(implicit ec: ExecutionContext, bodyParsers: PlayBodyParsers)
     extends Controller {
 
@@ -107,46 +105,29 @@ class DatasetController @Inject()(userService: UserService,
       }
     }
 
-  def exploreRemoteDataset(): Action[List[ExploreRemoteDatasetParameters]] =
-    sil.SecuredAction.async(validateJson[List[ExploreRemoteDatasetParameters]]) { implicit request =>
-      val reportMutable = ListBuffer[String]()
+  def exploreRemoteDataset(): Action[List[WKExploreRemoteLayerParameters]] =
+    sil.SecuredAction.async(validateJson[List[WKExploreRemoteLayerParameters]]) { implicit request =>
       for {
-        dataSourceBox: Box[GenericDataSource[DataLayer]] <- exploreRemoteLayerService
-          .exploreRemoteDatasource(request.body, request.identity, reportMutable)
-          .futureBox
-        dataSourceOpt = dataSourceBox match {
-          case Full(dataSource) if dataSource.dataLayers.nonEmpty =>
-            reportMutable += s"Resulted in dataSource with ${dataSource.dataLayers.length} layers."
-            Some(dataSource)
-          case Full(_) =>
-            reportMutable += "Error when exploring as layer set: Resulted in zero layers."
-            None
-          case f: Failure =>
-            reportMutable += s"Error when exploring as layer set: ${Fox.failureChainAsString(f)}"
-            None
-          case Empty =>
-            reportMutable += "Error when exploring as layer set: Empty"
-            None
-        }
-      } yield Ok(Json.obj("dataSource" -> Json.toJson(dataSourceOpt), "report" -> reportMutable.mkString("\n")))
+        exploreResponse <- wkExploreRemoteLayerService.exploreRemoteDatasource(request.body, request.identity)
+      } yield Ok(Json.toJson(exploreResponse))
     }
 
   // Note: This route is used by external applications, keep stable
   def exploreAndAddRemoteDataset(): Action[ExploreAndAddRemoteDatasetParameters] =
     sil.SecuredAction.async(validateJson[ExploreAndAddRemoteDatasetParameters]) { implicit request =>
-      val reportMutable = ListBuffer[String]()
-      val adaptedParameters = ExploreRemoteDatasetParameters(request.body.remoteUri, None, None, None)
+      val adaptedParameters =
+        WKExploreRemoteLayerParameters(request.body.remoteUri, None, None, None, request.body.dataStoreName)
       for {
-        dataSource <- exploreRemoteLayerService.exploreRemoteDatasource(List(adaptedParameters),
-                                                                        request.identity,
-                                                                        reportMutable)
+        exploreResponse <- wkExploreRemoteLayerService.exploreRemoteDatasource(List(adaptedParameters),
+                                                                               request.identity)
+        dataSource <- exploreResponse.dataSource ?~> "dataset.explore.failed"
         _ <- bool2Fox(dataSource.dataLayers.nonEmpty) ?~> "dataset.explore.zeroLayers"
         folderIdOpt <- Fox.runOptional(request.body.folderPath)(folderPath =>
           folderService.getOrCreateFromPathLiteral(folderPath, request.identity._organization)) ?~> "dataset.explore.autoAdd.getFolder.failed"
-        _ <- exploreRemoteLayerService.addRemoteDatasource(dataSource,
-                                                           request.body.datasetName,
-                                                           request.identity,
-                                                           folderIdOpt) ?~> "dataset.explore.autoAdd.failed"
+        _ <- wkExploreRemoteLayerService.addRemoteDatasource(dataSource,
+                                                             request.body.datasetName,
+                                                             request.identity,
+                                                             folderIdOpt) ?~> "dataset.explore.autoAdd.failed"
       } yield Ok
     }
 

diff --git a/app/models/dataset/WKRemoteDataStoreClient.scala b/app/models/dataset/WKRemoteDataStoreClient.scala
@@ -3,6 +3,11 @@ package models.dataset
 import com.scalableminds.util.cache.AlfuCache
 import com.scalableminds.util.geometry.{BoundingBox, Vec3Int}
 import com.scalableminds.util.tools.Fox
+import com.scalableminds.webknossos.datastore.explore.{
+  ExploreRemoteDatasetRequest,
+  ExploreRemoteDatasetResponse,
+  ExploreRemoteLayerParameters
+}
 import com.scalableminds.webknossos.datastore.models.{AdditionalCoordinate, RawCuboidRequest}
 import com.scalableminds.webknossos.datastore.models.datasource.{DataLayer, GenericDataSource}
 import com.scalableminds.webknossos.datastore.rpc.RPC
@@ -102,4 +107,12 @@ class WKRemoteDataStoreClient(dataStore: DataStore, rpc: RPC) extends LazyLoggin
     )
   }
 
+  def exploreRemoteDataset(layerParameters: List[ExploreRemoteLayerParameters],
+                           organizationName: String,
+                           userToken: String): Fox[ExploreRemoteDatasetResponse] =
+    rpc(s"${dataStore.url}/data/datasets/exploreRemote")
+      .addQueryString("token" -> userToken)
+      .postJsonWithJsonResponse[ExploreRemoteDatasetRequest, ExploreRemoteDatasetResponse](
+        ExploreRemoteDatasetRequest(layerParameters, organizationName))
+
 }
diff --git a/app/models/dataset/explore/ExploreRemoteLayerService.scala b/app/models/dataset/explore/ExploreRemoteLayerService.scala