scalableminds · bulldozer-boy · Jan 17, 2023 · Nov 15, 2022 · Nov 17, 2022 · Nov 17, 2022
diff --git a/CHANGELOG.unreleased.md b/CHANGELOG.unreleased.md
@@ -21,10 +21,11 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released
 - Bulk task creation now needs the taskTypeId, the task type summary will no longer be accepted. [#6640](https://github.com/scalableminds/webknossos/pull/6640)
 - Error handling and reporting is more robust now. [#6700](https://github.com/scalableminds/webknossos/pull/6700)
 - The Quick-Select settings are opened (and closed) automatically when labeling with the preview mode. That way, bulk labelings with preview mode don't require constantly opening the settings manually. [#6706](https://github.com/scalableminds/webknossos/pull/6706)
+- For remote datasets that require authentication, credentials are no longer stored in the respective JSON. [#6646](https://github.com/scalableminds/webknossos/pull/6646)
 
 ### Fixed
 - Fixed import of N5 datasets. [#6668](https://github.com/scalableminds/webknossos/pull/6668)
-- Fixed a bug where it was possible to create invalid an state by deleting teams that are referenced elsewhere. [6664](https://github.com/scalableminds/webknossos/pull/6664)
+- Fixed a bug where it was possible to create an invalid state by deleting teams that are referenced elsewhere. [6664](https://github.com/scalableminds/webknossos/pull/6664)
 - Miscellaneous fixes for the new folder UI. [#6674](https://github.com/scalableminds/webknossos/pull/6674)
 - Fixed import of remote datasets with multiple layers and differing resolution pyramid. #[6670](https://github.com/scalableminds/webknossos/pull/6670)
 - Fixed broken Get-new-Task button in task dashboard. [#6677](https://github.com/scalableminds/webknossos/pull/6677)

diff --git a/MIGRATIONS.unreleased.md b/MIGRATIONS.unreleased.md
@@ -15,3 +15,4 @@ User-facing changes are documented in the [changelog](CHANGELOG.released.md).
 - [091-folders.sql](conf/evolutions/091-folders.sql)
 - [092-oidc.sql](conf/evolutions/092-oidc.sql)
 - [093-terms-of-service.sql](conf/evolutions/093-terms-of-service.sql)
+- [094-credentials.sql](conf/evolutions/094-credentials.sql)
diff --git a/app/controllers/CredentialController.scala b/app/controllers/CredentialController.scala
@@ -0,0 +1,65 @@
+package controllers
+
+import com.mohiva.play.silhouette.api.Silhouette
+import com.scalableminds.util.tools.FoxImplicits
+import com.scalableminds.webknossos.datastore.storage.{HttpBasicAuthCredential, S3AccessKeyCredential}
+import models.binary.credential.CredentialDAO
+import oxalis.security.WkEnv
+import play.api.libs.json.{Json, OFormat}
+import play.api.mvc.{Action, PlayBodyParsers}
+import utils.ObjectId
+
+import javax.inject.Inject
+import scala.concurrent.ExecutionContext
+
+case class HttpBasicAuthCredentialParameters(name: String, username: String, password: String, domain: Option[String])
+
+object HttpBasicAuthCredentialParameters {
+  implicit val jsonFormat: OFormat[HttpBasicAuthCredentialParameters] = Json.format[HttpBasicAuthCredentialParameters]
+}
+
+case class S3AccessKeyCredentialParameters(name: String, keyId: String, key: String, bucket: Option[String])
+
+object S3AccessKeyCredentialParameters {
+  implicit val jsonFormat: OFormat[S3AccessKeyCredentialParameters] = Json.format[S3AccessKeyCredentialParameters]
+}
+
+class CredentialController @Inject()(credentialDAO: CredentialDAO, sil: Silhouette[WkEnv])(
+    implicit ec: ExecutionContext,
+    val bodyParsers: PlayBodyParsers)
+    extends Controller
+    with FoxImplicits {
+
+  def createHttpBasicAuthCredential: Action[HttpBasicAuthCredentialParameters] =
+    sil.SecuredAction.async(validateJson[HttpBasicAuthCredentialParameters]) { implicit request =>
+      val _id = ObjectId.generate
+      for {
+        _ <- bool2Fox(request.identity.isAdmin) ?~> "notAllowed" ~> FORBIDDEN
+        _ <- credentialDAO.insertOne(
+          _id,
+          HttpBasicAuthCredential(request.body.name,
+                                  request.body.username,
+                                  request.body.password,
+                                  request.identity._id.toString,
+                                  request.identity._organization.toString)
+        ) ?~> "create.failed"
+      } yield Ok(Json.toJson(_id))
+    }
+
+  def createS3AccessKeyCredential: Action[S3AccessKeyCredentialParameters] =
+    sil.SecuredAction.async(validateJson[S3AccessKeyCredentialParameters]) { implicit request =>
+      val _id = ObjectId.generate
+      for {
+        _ <- bool2Fox(request.identity.isAdmin) ?~> "notAllowed" ~> FORBIDDEN
+        _ <- credentialDAO.insertOne(
+          _id,
+          S3AccessKeyCredential(request.body.name,
+                                request.body.keyId,
+                                request.body.key,
+                                request.identity._id.toString,
+                                request.identity._organization.toString)
+        ) ?~> "create.failed"
+      } yield Ok(Json.toJson(_id))
+    }
+
+}
diff --git a/app/controllers/DataSetController.scala b/app/controllers/DataSetController.scala
@@ -136,7 +136,7 @@ class DataSetController @Inject()(userService: UserService,
       val reportMutable = ListBuffer[String]()
       for {
         dataSourceBox: Box[GenericDataSource[DataLayer]] <- exploreRemoteLayerService
-          .exploreRemoteDatasource(request.body, reportMutable)
+          .exploreRemoteDatasource(request.body, request.identity, reportMutable)
           .futureBox
         dataSourceOpt = dataSourceBox match {
           case Full(dataSource) if dataSource.dataLayers.nonEmpty =>

diff --git a/app/controllers/WKRemoteDataStoreController.scala b/app/controllers/WKRemoteDataStoreController.scala
@@ -11,9 +11,11 @@ import com.scalableminds.webknossos.datastore.services.{
   ReserveUploadInformation
 }
 import com.typesafe.scalalogging.LazyLogging
+
 import javax.inject.Inject
 import models.analytics.{AnalyticsService, UploadDatasetEvent}
 import models.binary._
+import models.binary.credential.CredentialDAO
 import models.job.JobDAO
 import models.organization.OrganizationDAO
 import models.user.{User, UserDAO, UserService}
@@ -37,6 +39,7 @@ class WKRemoteDataStoreController @Inject()(
     dataSetDAO: DataSetDAO,
     userDAO: UserDAO,
     jobDAO: JobDAO,
+    credentialDAO: CredentialDAO,
     mailchimpClient: MailchimpClient,
     wkSilhouetteEnvironment: WkSilhouetteEnvironment)(implicit ec: ExecutionContext, bodyParsers: PlayBodyParsers)
     extends Controller
@@ -176,4 +179,14 @@ class WKRemoteDataStoreController @Inject()(
       }
   }
 
+  def findCredential(name: String, key: String, credentialId: String): Action[AnyContent] = Action.async {
+    implicit request =>
+      dataStoreService.validateAccess(name, key) { _ =>
+        for {
+          credentialIdValidated <- ObjectId.fromString(credentialId)
+          credential <- credentialDAO.findOne(credentialIdValidated)
+        } yield Ok(Json.toJson(credential))
+      }
+  }
+
 }
diff --git a/app/models/binary/credential/Credential.scala b/app/models/binary/credential/Credential.scala
@@ -0,0 +1,90 @@
+package models.binary.credential
+
+import com.scalableminds.util.tools.Fox
+import com.scalableminds.webknossos.datastore.storage.{AnyCredential, HttpBasicAuthCredential, S3AccessKeyCredential}
+import com.scalableminds.webknossos.schema.Tables
+import com.scalableminds.webknossos.schema.Tables.{Credentials, CredentialsRow}
+import slick.jdbc.PostgresProfile.api._
+import slick.lifted.Rep
+import utils.{ObjectId, SQLClient, SQLDAO}
+
+import javax.inject.Inject
+import scala.concurrent.ExecutionContext
+
+// Generic credential as it appears in the database
+case class Credential(_id: ObjectId,
+                      credentialType: CredentialType.Value,
+                      name: String,
+                      identifier: Option[String],
+                      secret: Option[String],
+                      scope: Option[String],
+                      filePath: Option[String])
+
+class CredentialDAO @Inject()(sqlClient: SQLClient)(implicit ec: ExecutionContext)
+    extends SQLDAO[Credential, CredentialsRow, Credentials](sqlClient) {
+  protected val collection = Credentials
+
+  protected def idColumn(x: Credentials): Rep[String] = x._Id
+  override protected def isDeletedColumn(x: Tables.Credentials): Rep[Boolean] = x.isdeleted
+
+  // use parseAnyCredential instead
+  protected def parse(row: com.scalableminds.webknossos.schema.Tables.Credentials#TableElementType)
+    : com.scalableminds.util.tools.Fox[models.binary.credential.Credential] = ???
+
+  private def parseAsHttpBasicAuthCredential(r: CredentialsRow): Fox[HttpBasicAuthCredential] =
+    for {
+      username <- r.identifier.toFox
+      password <- r.secret.toFox
+    } yield
+      HttpBasicAuthCredential(
+        r.name,
+        username,
+        password,
+        r._User,
+        r._Organization
+      )
+
+  private def parseAsS3AccessKeyCredential(r: CredentialsRow): Fox[S3AccessKeyCredential] =
+    for {
+      keyId <- r.identifier.toFox
+      key <- r.secret.toFox
+    } yield
+      S3AccessKeyCredential(
+        r.name,
+        keyId,
+        key,
+        r._User,
+        r._Organization
+      )
+
+  def insertOne(_id: ObjectId, credential: HttpBasicAuthCredential): Fox[Unit] =
+    for {
+      _ <- run(sqlu"""insert into webknossos.credentials(_id, type, name, identifier, secret, _user, _organization)
+                     values(${_id}, '#${CredentialType.HTTP_Basic_Auth}', ${credential.name}, ${credential.username}, ${credential.password}, ${credential.user}, ${credential.organization})""")
+    } yield ()
+
+  def insertOne(_id: ObjectId, credential: S3AccessKeyCredential): Fox[Unit] =
+    for {
+      _ <- run(sqlu"""insert into webknossos.credentials(_id, type, name, identifier, secret, _user, _organization)
+                     values(${_id}, '#${CredentialType.S3_Access_Key}', ${credential.name}, ${credential.keyId}, ${credential.key}, ${credential.user}, ${credential.organization})""")
+    } yield ()
+
+  def findOne(id: ObjectId): Fox[AnyCredential] =
+    for {
+      r <- run(sql"select #$columns from webknossos.credentials_ where _id = $id".as[CredentialsRow])
+      firstRow <- r.headOption.toFox
+      parsed <- parseAnyCredential(firstRow)
+    } yield parsed
+
+  private def parseAnyCredential(r: CredentialsRow): Fox[AnyCredential] =
+    r.`type` match {
+      case "HTTP_Basic_Auth" =>
+        for {
+          parsed <- parseAsHttpBasicAuthCredential(r)
+        } yield parsed
+      case "S3_Access_Key" =>
+        for {
+          parsed <- parseAsS3AccessKeyCredential(r)
+        } yield parsed
+    }
+}
diff --git a/app/models/binary/credential/CredentialService.scala b/app/models/binary/credential/CredentialService.scala
@@ -0,0 +1,51 @@
+package models.binary.credential
+
+import com.scalableminds.util.tools.Fox
+import com.scalableminds.webknossos.datastore.storage.{HttpBasicAuthCredential, S3AccessKeyCredential}
+import utils.ObjectId
+
+import java.net.URI
+import javax.inject.Inject
+import scala.concurrent.ExecutionContext
+
+class CredentialService @Inject()(credentialDao: CredentialDAO) {
+
+  def createCredential(uri: URI,
+                       username: Option[String],
+                       password: Option[String],
+                       user: String,
+                       organization: String)(implicit ec: ExecutionContext): Fox[Option[ObjectId]] = {
+    val scheme = uri.getScheme
+    scheme match {
+      case "https" =>
+        username match {
+          case Some(u) =>
+            val _id = ObjectId.generate
+            for {
+              _ <- credentialDao.insertOne(
+                _id,
+                HttpBasicAuthCredential(uri.toString, u, password.getOrElse(""), user, organization))
+              _ <- credentialDao.findOne(_id)
+            } yield Some(_id)
+          case None => Fox.empty
+        }
+      case "s3" =>
+        username match {
+          case Some(keyId) =>
+            password match {
+              case Some(secretKey) =>
+                val _id = ObjectId.generate
+                for {
+                  _ <- credentialDao.insertOne(
+                    _id,
+                    S3AccessKeyCredential(uri.toString, keyId, secretKey, user, organization))
+                  _ <- credentialDao.findOne(_id)
+                } yield Some(_id)
+              case None => Fox.empty
+            }
+          case None => Fox.empty
+        }
+    }
+  }
+
+}
diff --git a/app/models/binary/credential/CredentialType.scala b/app/models/binary/credential/CredentialType.scala
@@ -0,0 +1,9 @@
+package models.binary.credential
+
+import com.scalableminds.util.enumeration.ExtendedEnumeration
+
+object CredentialType extends ExtendedEnumeration {
+  type CredentialType = Value
+
+  val HTTP_Basic_Auth, S3_Access_Key, HTTP_Token, GCS = Value
+}
diff --git a/app/models/binary/explore/ExploreRemoteLayerService.scala b/app/models/binary/explore/ExploreRemoteLayerService.scala
@@ -9,8 +9,10 @@ import com.scalableminds.webknossos.datastore.datareaders.zarr._
 import com.scalableminds.webknossos.datastore.models.datasource._
 import com.scalableminds.webknossos.datastore.storage.FileSystemsHolder
 import com.typesafe.scalalogging.LazyLogging
+import models.binary.credential.CredentialService
 import net.liftweb.common.{Box, Empty, Failure, Full}
 import net.liftweb.util.Helpers.tryo
+import oxalis.security.WkEnv
 import play.api.libs.json.{Json, OFormat}
 
 import java.net.URI
@@ -26,14 +28,20 @@ object ExploreRemoteDatasetParameters {
   implicit val jsonFormat: OFormat[ExploreRemoteDatasetParameters] = Json.format[ExploreRemoteDatasetParameters]
 }
 
-class ExploreRemoteLayerService @Inject()() extends FoxImplicits with LazyLogging {
+class ExploreRemoteLayerService @Inject()(credentialService: CredentialService) extends FoxImplicits with LazyLogging {
 
   def exploreRemoteDatasource(
       urisWithCredentials: List[ExploreRemoteDatasetParameters],
+      requestIdentity: WkEnv#I,
       reportMutable: ListBuffer[String])(implicit ec: ExecutionContext): Fox[GenericDataSource[DataLayer]] =
     for {
-      exploredLayersNested <- Fox.serialCombined(urisWithCredentials)(parameters =>
-        exploreRemoteLayersForUri(parameters.remoteUri, parameters.user, parameters.password, reportMutable))
+      exploredLayersNested <- Fox.serialCombined(urisWithCredentials)(
+        parameters =>
+          exploreRemoteLayersForUri(parameters.remoteUri,
+                                    parameters.user,
+                                    parameters.password,
+                                    reportMutable,
+                                    requestIdentity))
       layersWithVoxelSizes = exploredLayersNested.flatten
       _ <- bool2Fox(layersWithVoxelSizes.nonEmpty) ?~> "Detected zero layers"
       rescaledLayersAndVoxelSize <- rescaleLayersByCommonVoxelSize(layersWithVoxelSizes) ?~> "Could not extract common voxel size from layers"
@@ -131,14 +139,20 @@ class ExploreRemoteLayerService @Inject()() extends FoxImplicits with LazyLoggin
       layerUri: String,
       user: Option[String],
       password: Option[String],
-      reportMutable: ListBuffer[String])(implicit ec: ExecutionContext): Fox[List[(DataLayer, Vec3Double)]] =
+      reportMutable: ListBuffer[String],
+      requestIdentity: WkEnv#I)(implicit ec: ExecutionContext): Fox[List[(DataLayer, Vec3Double)]] =
     for {
       remoteSource <- tryo(RemoteSourceDescriptor(new URI(normalizeUri(layerUri)), user, password)).toFox ?~> s"Received invalid URI: $layerUri"
+      credentialId <- credentialService.createCredential(new URI(normalizeUri(layerUri)),
+                                                         user,
+                                                         password,
+                                                         requestIdentity._id.toString,
+                                                         requestIdentity._organization.toString)
       fileSystem <- FileSystemsHolder.getOrCreate(remoteSource).toFox ?~> "Failed to set up remote file system"
       remotePath <- tryo(fileSystem.getPath(remoteSource.remotePath)) ?~> "Failed to get remote path"
       layersWithVoxelSizes <- exploreRemoteLayersForRemotePath(
         remotePath,
-        remoteSource.credentials,
+        credentialId.map(o => o.toString),
         reportMutable,
         List(new ZarrArrayExplorer, new NgffExplorer, new N5ArrayExplorer, new N5MultiscalesExplorer))
     } yield layersWithVoxelSizes
@@ -153,23 +167,23 @@ class ExploreRemoteLayerService @Inject()() extends FoxImplicits with LazyLoggin
 
   private def exploreRemoteLayersForRemotePath(
       remotePath: Path,
-      credentials: Option[FileSystemCredentials],
+      credentialId: Option[String],
       reportMutable: ListBuffer[String],
       explorers: List[RemoteLayerExplorer])(implicit ec: ExecutionContext): Fox[List[(DataLayer, Vec3Double)]] =
     explorers match {
       case Nil => Fox.empty
       case currentExplorer :: remainingExplorers =>
         reportMutable += s"\nTrying to explore $remotePath as ${currentExplorer.name}..."
-        currentExplorer.explore(remotePath, credentials).futureBox.flatMap {
+        currentExplorer.explore(remotePath, credentialId).futureBox.flatMap {
           case Full(layersWithVoxelSizes) =>
             reportMutable += s"Found ${layersWithVoxelSizes.length} ${currentExplorer.name} layers at $remotePath."
             Fox.successful(layersWithVoxelSizes)
           case f: Failure =>
             reportMutable += s"Error when reading $remotePath as ${currentExplorer.name}: ${formatFailureForReport(f)}"
-            exploreRemoteLayersForRemotePath(remotePath, credentials, reportMutable, remainingExplorers)
+            exploreRemoteLayersForRemotePath(remotePath, credentialId, reportMutable, remainingExplorers)
           case Empty =>
             reportMutable += s"Error when reading $remotePath as ${currentExplorer.name}: Empty"
-            exploreRemoteLayersForRemotePath(remotePath, credentials, reportMutable, remainingExplorers)
+            exploreRemoteLayersForRemotePath(remotePath, credentialId, reportMutable, remainingExplorers)
         }
     }
 

diff --git a/app/models/binary/explore/N5ArrayExplorer.scala b/app/models/binary/explore/N5ArrayExplorer.scala
@@ -3,7 +3,6 @@ import com.scalableminds.util.geometry.{Vec3Double, Vec3Int}
 import com.scalableminds.util.tools.Fox
 import com.scalableminds.webknossos.datastore.dataformats.MagLocator
 import com.scalableminds.webknossos.datastore.dataformats.n5.{N5DataLayer, N5Layer, N5SegmentationLayer}
-import com.scalableminds.webknossos.datastore.dataformats.zarr.FileSystemCredentials
 import com.scalableminds.webknossos.datastore.datareaders.AxisOrder
 import com.scalableminds.webknossos.datastore.datareaders.n5.N5Header
 import com.scalableminds.webknossos.datastore.models.datasource.Category
@@ -15,15 +14,15 @@ class N5ArrayExplorer extends RemoteLayerExplorer {
 
   override def name: String = "N5 Array"
 
-  override def explore(remotePath: Path, credentials: Option[FileSystemCredentials]): Fox[List[(N5Layer, Vec3Double)]] =
+  override def explore(remotePath: Path, credentialId: Option[String]): Fox[List[(N5Layer, Vec3Double)]] =
     for {
       headerPath <- Fox.successful(remotePath.resolve(N5Header.FILENAME_ATTRIBUTES_JSON))
       name <- guessNameFromPath(remotePath)
       n5Header <- parseJsonFromPath[N5Header](headerPath) ?~> s"failed to read n5 header at $headerPath"
       elementClass <- n5Header.elementClass ?~> "failed to read element class from n5 header"
       guessedAxisOrder = AxisOrder.asZyxFromRank(n5Header.rank)
       boundingBox <- n5Header.boundingBox(guessedAxisOrder) ?~> "failed to read bounding box from zarr header. Make sure data is in (T/C)ZYX format"
-      magLocator = MagLocator(Vec3Int.ones, Some(remotePath.toString), credentials, Some(guessedAxisOrder), None)
+      magLocator = MagLocator(Vec3Int.ones, Some(remotePath.toString), None, Some(guessedAxisOrder), None, credentialId)
       layer: N5Layer = if (looksLikeSegmentationLayer(name, elementClass)) {
         N5SegmentationLayer(name, boundingBox, elementClass, List(magLocator), largestSegmentId = None)
       } else N5DataLayer(name, Category.color, boundingBox, elementClass, List(magLocator))