-
Notifications
You must be signed in to change notification settings - Fork 24
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Store remote dataset credentials separately (#6646)
* Add schema for credentials * Add credential model * Simplify AnyCredential * Rename httpbasicauth * Add routes for credential creation * Fix compilation * Add internal route to get credential * Fix compilation error * Fix things * [wip] pass injected fileSystemService to BucketProvider * Merge branch 'master' into store-credentials-separately * propagate execution context to bucket loading, change signature to fox * Fetch credentials from db for maglocator * Remove remote source from maglocator * Make scope optional * Merge, bump schema Version to 93 * Remove unused import * Merge branch 'master' into store-credentials-separately * Return credential id on credential creation * Make request silent * Merge branch 'master' into store-credentials-separately * add bucket provider cache * format * Allow credentials to be created when exploring * Update schema to include user, org, isDeleted * Change backend format command * Merge branch 'master' into store-credentials-separately * Fix usage of legacy credentials and no credentials * Fix schema * Merge branch 'master' into store-credentials-separately * Update schema number and migrations, changelog * Make match exhaustive * Use protected and private * Merge branch 'master' into store-credentials-separately * Merge branch 'master' into store-credentials-separately * Use SecuredSQLDAO instead of SQLDAO * Merge branch 'master' into store-credentials-separately * Update schema number * Merge branch 'master' into store-credentials-separately * Fix error in findOne * Merge branch 'master' into store-credentials-separately
- Loading branch information
Showing
39 changed files
with
528 additions
and
152 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,65 @@ | ||
| package controllers | ||
|
|
||
| import com.mohiva.play.silhouette.api.Silhouette | ||
| import com.scalableminds.util.tools.FoxImplicits | ||
| import com.scalableminds.webknossos.datastore.storage.{HttpBasicAuthCredential, S3AccessKeyCredential} | ||
| import models.binary.credential.CredentialDAO | ||
| import oxalis.security.WkEnv | ||
| import play.api.libs.json.{Json, OFormat} | ||
| import play.api.mvc.{Action, PlayBodyParsers} | ||
| import utils.ObjectId | ||
|
|
||
| import javax.inject.Inject | ||
| import scala.concurrent.ExecutionContext | ||
|
|
||
| case class HttpBasicAuthCredentialParameters(name: String, username: String, password: String, domain: Option[String]) | ||
|
|
||
| object HttpBasicAuthCredentialParameters { | ||
| implicit val jsonFormat: OFormat[HttpBasicAuthCredentialParameters] = Json.format[HttpBasicAuthCredentialParameters] | ||
| } | ||
|
|
||
| case class S3AccessKeyCredentialParameters(name: String, keyId: String, key: String, bucket: Option[String]) | ||
|
|
||
| object S3AccessKeyCredentialParameters { | ||
| implicit val jsonFormat: OFormat[S3AccessKeyCredentialParameters] = Json.format[S3AccessKeyCredentialParameters] | ||
| } | ||
|
|
||
| class CredentialController @Inject()(credentialDAO: CredentialDAO, sil: Silhouette[WkEnv])( | ||
| implicit ec: ExecutionContext, | ||
| val bodyParsers: PlayBodyParsers) | ||
| extends Controller | ||
| with FoxImplicits { | ||
|
|
||
| def createHttpBasicAuthCredential: Action[HttpBasicAuthCredentialParameters] = | ||
| sil.SecuredAction.async(validateJson[HttpBasicAuthCredentialParameters]) { implicit request => | ||
| val _id = ObjectId.generate | ||
| for { | ||
| _ <- bool2Fox(request.identity.isAdmin) ?~> "notAllowed" ~> FORBIDDEN | ||
| _ <- credentialDAO.insertOne( | ||
| _id, | ||
| HttpBasicAuthCredential(request.body.name, | ||
| request.body.username, | ||
| request.body.password, | ||
| request.identity._id.toString, | ||
| request.identity._organization.toString) | ||
| ) ?~> "create.failed" | ||
| } yield Ok(Json.toJson(_id)) | ||
| } | ||
|
|
||
| def createS3AccessKeyCredential: Action[S3AccessKeyCredentialParameters] = | ||
| sil.SecuredAction.async(validateJson[S3AccessKeyCredentialParameters]) { implicit request => | ||
| val _id = ObjectId.generate | ||
| for { | ||
| _ <- bool2Fox(request.identity.isAdmin) ?~> "notAllowed" ~> FORBIDDEN | ||
| _ <- credentialDAO.insertOne( | ||
| _id, | ||
| S3AccessKeyCredential(request.body.name, | ||
| request.body.keyId, | ||
| request.body.key, | ||
| request.identity._id.toString, | ||
| request.identity._organization.toString) | ||
| ) ?~> "create.failed" | ||
| } yield Ok(Json.toJson(_id)) | ||
| } | ||
|
|
||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,69 @@ | ||
| package models.binary.credential | ||
|
|
||
| import com.scalableminds.util.tools.Fox | ||
| import com.scalableminds.webknossos.datastore.storage.{AnyCredential, HttpBasicAuthCredential, S3AccessKeyCredential} | ||
| import com.scalableminds.webknossos.schema.Tables.{Credentials, CredentialsRow} | ||
| import utils.sql.{SecuredSQLDAO, SqlClient, SqlToken} | ||
| import utils.ObjectId | ||
|
|
||
| import javax.inject.Inject | ||
| import scala.concurrent.ExecutionContext | ||
|
|
||
| class CredentialDAO @Inject()(sqlClient: SqlClient)(implicit ec: ExecutionContext) extends SecuredSQLDAO(sqlClient) { | ||
| protected val collection = Credentials | ||
|
|
||
| protected def columnsList: List[String] = collection.baseTableRow.create_*.map(_.name).toList | ||
| override protected def collectionName: String = "credentials" | ||
| def columns: SqlToken = SqlToken.raw(columnsList.mkString(", ")) | ||
|
|
||
| private def parseAsHttpBasicAuthCredential(r: CredentialsRow): Fox[HttpBasicAuthCredential] = | ||
| for { | ||
| username <- r.identifier.toFox | ||
| password <- r.secret.toFox | ||
| } yield | ||
| HttpBasicAuthCredential( | ||
| r.name, | ||
| username, | ||
| password, | ||
| r._User, | ||
| r._Organization | ||
| ) | ||
|
|
||
| private def parseAsS3AccessKeyCredential(r: CredentialsRow): Fox[S3AccessKeyCredential] = | ||
| for { | ||
| keyId <- r.identifier.toFox | ||
| key <- r.secret.toFox | ||
| } yield | ||
| S3AccessKeyCredential( | ||
| r.name, | ||
| keyId, | ||
| key, | ||
| r._User, | ||
| r._Organization | ||
| ) | ||
|
|
||
| def insertOne(_id: ObjectId, credential: HttpBasicAuthCredential): Fox[Unit] = | ||
| for { | ||
| _ <- run(q"""insert into webknossos.credentials(_id, type, name, identifier, secret, _user, _organization) | ||
| values(${_id}, ${CredentialType.HTTP_Basic_Auth}, ${credential.name}, ${credential.username}, ${credential.password}, ${credential.user}, ${credential.organization})""".asUpdate) | ||
| } yield () | ||
|
|
||
| def insertOne(_id: ObjectId, credential: S3AccessKeyCredential): Fox[Unit] = | ||
| for { | ||
| _ <- run(q"""insert into webknossos.credentials(_id, type, name, identifier, secret, _user, _organization) | ||
| values(${_id}, ${CredentialType.S3_Access_Key}, ${credential.name}, ${credential.keyId}, ${credential.key}, ${credential.user}, ${credential.organization})""".asUpdate) | ||
| } yield () | ||
|
|
||
| def findOne(id: ObjectId): Fox[AnyCredential] = | ||
| for { | ||
| r <- run(q"select $columns from webknossos.credentials_ where _id = $id".as[CredentialsRow]) | ||
| firstRow <- r.headOption.toFox | ||
| parsed <- parseAnyCredential(firstRow) | ||
| } yield parsed | ||
|
|
||
| private def parseAnyCredential(r: CredentialsRow): Fox[AnyCredential] = | ||
| r.`type` match { | ||
| case "HTTP_Basic_Auth" => parseAsHttpBasicAuthCredential(r) | ||
| case "S3_Access_Key" => parseAsS3AccessKeyCredential(r) | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,51 @@ | ||
| package models.binary.credential | ||
|
|
||
| import com.scalableminds.util.tools.Fox | ||
| import com.scalableminds.webknossos.datastore.storage.{HttpBasicAuthCredential, S3AccessKeyCredential} | ||
| import utils.ObjectId | ||
|
|
||
| import java.net.URI | ||
| import javax.inject.Inject | ||
| import scala.concurrent.ExecutionContext | ||
|
|
||
| class CredentialService @Inject()(credentialDao: CredentialDAO) { | ||
|
|
||
| def createCredential(uri: URI, | ||
| username: Option[String], | ||
| password: Option[String], | ||
| user: String, | ||
| organization: String)(implicit ec: ExecutionContext): Fox[Option[ObjectId]] = { | ||
| val scheme = uri.getScheme | ||
| scheme match { | ||
| case "https" => | ||
| username match { | ||
| case Some(u) => | ||
| val _id = ObjectId.generate | ||
| for { | ||
| _ <- credentialDao.insertOne( | ||
| _id, | ||
| HttpBasicAuthCredential(uri.toString, u, password.getOrElse(""), user, organization)) | ||
| _ <- credentialDao.findOne(_id) | ||
| } yield Some(_id) | ||
| case None => Fox.empty | ||
| } | ||
| case "s3" => | ||
| username match { | ||
| case Some(keyId) => | ||
| password match { | ||
| case Some(secretKey) => | ||
| val _id = ObjectId.generate | ||
| for { | ||
| _ <- credentialDao.insertOne( | ||
| _id, | ||
| S3AccessKeyCredential(uri.toString, keyId, secretKey, user, organization)) | ||
| _ <- credentialDao.findOne(_id) | ||
| } yield Some(_id) | ||
| case None => Fox.empty | ||
| } | ||
| case None => Fox.empty | ||
| } | ||
| } | ||
| } | ||
|
|
||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,9 @@ | ||
| package models.binary.credential | ||
|
|
||
| import com.scalableminds.util.enumeration.ExtendedEnumeration | ||
|
|
||
| object CredentialType extends ExtendedEnumeration { | ||
| type CredentialType = Value | ||
|
|
||
| val HTTP_Basic_Auth, S3_Access_Key, HTTP_Token, GCS = Value | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.