Allow deleting datasets on disk

CHANGELOG.unreleased.md

@@ -15,6 +15,7 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released
 - Added a warning to the segmentation tab when viewing `uint64` bit segmentation data. [#4598](https://github.com/scalableminds/webknossos/pull/4598)
 - Added the possibility to have multiple user-defined bounding boxes in an annotation. Task bounding boxes are automatically converted to such user bounding boxes upon “copy to my account” / reupload as explorational annotation. [#4536](https://github.com/scalableminds/webknossos/pull/4536)
 - Added additional information to each task in CSV download. [#4647](https://github.com/scalableminds/webknossos/pull/4647)
+- Added the possibility to delete datasets on disk from webKnossos. Use with care. [#4696]()(https://github.com/scalableminds/webknossos/pull/4696)
 
 ### Changed
 - Separated the permissions of Team Managers (now actually team-scoped) and Dataset Managers (who can see all datasets). The database evolution makes all Team Managers also Dataset Managers, so no workflows should be interrupted. New users may have to be made Dataset Managers, though. For more information, refer to the updated documentation. [#4663](https://github.com/scalableminds/webknossos/pull/4663)

app/controllers/UserTokenController.scala

@@ -101,7 +101,7 @@ class UserTokenController @Inject()(dataSetDAO: DataSetDAO,
     def tryWrite: Fox[UserAccessAnswer] =
       for {
         dataset <- dataSetDAO.findOneByNameAndOrganizationName(dataSourceId.name, dataSourceId.team) ?~> "datasource.notFound"
-        user <- userBox.toFox
+        user <- userBox.toFox ?~> "auth.token.noUser"
         isAllowed <- dataSetService.isEditableBy(dataset, Some(user))
       } yield {
         UserAccessAnswer(isAllowed)
@@ -116,10 +116,18 @@ class UserTokenController @Inject()(dataSetDAO: DataSetDAO,
         case _ => Fox.successful(UserAccessAnswer(false, Some("invalid access token")))
       }
 
+    def tryDelete: Fox[UserAccessAnswer] =
+      for {
+        dataset <- dataSetDAO.findOneByNameAndOrganizationName(dataSourceId.name, dataSourceId.team)(
+          GlobalAccessContext) ?~> "datasource.notFound"
+        user <- userBox.toFox ?~> "auth.token.noUser"
+      } yield UserAccessAnswer(user._organization == dataset._organization && user.isAdmin)
+
     mode match {
       case AccessMode.read         => tryRead
       case AccessMode.write        => tryWrite
       case AccessMode.administrate => tryAdministrate
+      case AccessMode.delete       => tryDelete
       case _                       => Fox.successful(UserAccessAnswer(false, Some("invalid access token")))
     }
   }

conf/messages

@@ -32,6 +32,7 @@ image.create.failed=Failed to create image
 
 auth.tokenDeleted=Token was deleted
 auth.invalidToken=The token is invalid
+auth.token.noUser=Could not determine user for access token
 
 team.created=Team was created
 team.notAllowed=You are not part of this team. Project can’t be created.

frontend/javascripts/admin/admin_rest_api.js

@@ -915,6 +915,23 @@ export async function triggerDatasetClearCache(
   );
 }
 
+export async function deleteDatasetOnDisk(
+  datastoreHost: string,
+  datasetId: APIDatasetId,
+): Promise<void> {
+  await doWithToken(token =>
+    Request.triggerRequest(
+      `/data/datasets/${datasetId.owningOrganization}/${
+        datasetId.name
+      }/deleteOnDisk?token=${token}`,
+      {
+        host: datastoreHost,
+        method: "DELETE",
+      },
+    ),
+  );
+}
+
 export async function triggerDatasetClearThumbnailCache(datasetId: APIDatasetId): Promise<void> {
   await Request.triggerRequest(
     `/api/datasets/${datasetId.owningOrganization}/${datasetId.name}/clearThumbnailCache`,

frontend/javascripts/dashboard/dataset/dataset_import_view.js

@@ -29,6 +29,7 @@ import { Hideable, confirmAsync, hasFormError, jsonEditStyle } from "./helper_co
 import DefaultConfigComponent from "./default_config_component";
 import ImportGeneralComponent from "./import_general_component";
 import ImportSharingComponent from "./import_sharing_component";
+import ImportDeleteComponent from "./import_delete_component";
 import SimpleAdvancedDataForm from "./simple_advanced_data_form";
 
 const { TabPane } = Tabs;
@@ -539,6 +540,12 @@ class DatasetImportView extends React.PureComponent<Props, State> {
                     <DefaultConfigComponent form={form} />
                   </Hideable>
                 </TabPane>
+
+                <TabPane tab={<span> Delete Dataset </span>} key="deleteDataset" forceRender>
+                  <Hideable hidden={this.state.activeTabKey !== "deleteDataset"}>
+                    <ImportDeleteComponent datasetId={this.props.datasetId} />
+                  </Hideable>
+                </TabPane>
               </Tabs>
             </Card>
             <FormItem>

frontend/javascripts/dashboard/dataset/import_delete_component.js

@@ -0,0 +1,61 @@
+// @flow
+
+import { Button } from "antd";
+import React, { useState, useEffect } from "react";
+import * as Utils from "libs/utils";
+
+import type { APIDataset, APIDatasetId } from "admin/api_flow_types";
+import { getDataset, deleteDatasetOnDisk } from "admin/admin_rest_api";
+import Toast from "libs/toast";
+import messages from "messages";
+
+import { confirmAsync } from "./helper_components";
+
+type Props = {
+  datasetId: APIDatasetId,
+};
+
+export default function ImportSharingComponent({ datasetId }: Props) {
+  const [isDeleting, setIsDeleting] = useState(false);
+  const [dataSet, setDataSet] = useState<?APIDataset>(null);
+
+  async function fetch() {
+    const newDataSet = await getDataset(datasetId);
+    setDataSet(newDataSet);
+  }
+
+  useEffect(() => {
+    fetch();
+  }, []);
+
+  async function handleDeleteButtonClicked(): Promise<void> {
+    if (!dataSet) {
+      return;
+    }
+    await confirmAsync({
+      title: "Deleting a dataset on disk cannot be undone. Are you certain?",
+      okText: "Yes, Delete Dataset on Disk now",
+    });
+    setIsDeleting(true);
+    await deleteDatasetOnDisk(dataSet.dataStore.url, datasetId);
+    Toast.success(
+      messages["dataset.delete_success"]({
+        datasetName: dataSet.name,
+      }),
+    );
+    await Utils.sleep(2000);
+    setIsDeleting(false);
+    location.href = "/dashboard";
+  }
+
+  return (
+    <div>
+      <p>Deleting a dataset on disk cannot be undone. Please be certain.</p>
+      <p>Note that annotations for the dataset stay downloadable and the name stays reserved.</p>
+      <p>Only admins are allowed to delete datasets.</p>
+      <Button type="danger" loading={isDeleting} onClick={handleDeleteButtonClicked}>
+        Delete Dataset on Disk
+      </Button>
+    </div>
+  );
+}

frontend/javascripts/messages.js

@@ -158,6 +158,9 @@ instead. Only enable this option if you understand its effect. All layers will n
   "dataset.clear_cache_success": _.template(
     "The dataset <%- datasetName %> was reloaded successfully.",
   ),
+  "dataset.delete_success": _.template(
+    "The dataset <%- datasetName %> was successfully deleted on disk. Redirecting to dashboard...",
+  ),
   "task.no_tasks_to_download": "There are no tasks available to download.",
   "dataset.upload_success": "The dataset was uploaded successfully.",
   "dataset.add_success": "The dataset was added successfully.",

webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DataSourceController.scala

@@ -233,4 +233,14 @@ class DataSourceController @Inject()(
       }
   }
 
+  def deleteOnDisk(organizationName: String, dataSetName: String) = Action.async { implicit request =>
+    accessTokenService.validateAccess(UserAccessRequest.deleteDataSource(DataSourceId(dataSetName, organizationName))) {
+      AllowRemoteOrigin {
+        for {
+          _ <- binaryDataServiceHolder.binaryDataService.deleteOnDisk(organizationName, dataSetName)
+        } yield Ok
+      }
+    }
+  }
+
 }

webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AccessTokenService.scala

@@ -13,7 +13,7 @@ import scala.concurrent.duration._
 
 object AccessMode extends Enumeration {
 
-  val administrate, list, read, write = Value
+  val administrate, list, read, write, delete = Value
 
   implicit val jsonFormat = Format(Reads.enumNameReads(AccessMode), Writes.enumNameWrites)
 }
@@ -35,6 +35,8 @@ object UserAccessAnswer { implicit val jsonFormat = Json.format[UserAccessAnswer
 object UserAccessRequest {
   implicit val jsonFormat = Json.format[UserAccessRequest]
 
+  def deleteDataSource(dataSourceId: DataSourceId) =
+    UserAccessRequest(dataSourceId, AccessResourceType.datasource, AccessMode.delete)
   def administrateDataSources =
     UserAccessRequest(DataSourceId("", ""), AccessResourceType.datasource, AccessMode.administrate)
   def listDataSources =

webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/BinaryDataService.scala

@@ -1,7 +1,8 @@
 package com.scalableminds.webknossos.datastore.services
-
+import scala.reflect.io.Directory
+import java.io.File
 import java.nio.{ByteBuffer, ByteOrder, LongBuffer}
-import java.nio.file.{Path, Paths}
+import java.nio.file.{Files, Path, Paths, StandardCopyOption}
 
 import com.scalableminds.util.geometry.{Point3D, Vector3I}
 import com.scalableminds.webknossos.datastore.models.BucketPosition
@@ -21,6 +22,7 @@ import com.scalableminds.webknossos.datastore.storage.{
 import com.scalableminds.util.tools.ExtendedTypes.ExtendedArraySeq
 import com.scalableminds.util.tools.{Fox, FoxImplicits}
 import com.typesafe.scalalogging.LazyLogging
+import net.liftweb.common.Full
 import spire.math.UInt
 
 import scala.collection.mutable
@@ -201,4 +203,27 @@ class BinaryDataService(dataBaseDir: Path,
     agglomerateService.cache.clear(matchingAgglomerate)
     cache.clear(matchingPredicate)
   }
+
+  def deleteOnDisk(organizationName: String, dataSetName: String): Fox[Unit] = {
+    val dataSourcePath = dataBaseDir.resolve(organizationName).resolve(dataSetName)
+    val trashPath: Path = dataBaseDir.resolve(organizationName).resolve(".trash")
+    val targetPath = trashPath.resolve(dataSetName)
+    new File(trashPath.toString).mkdirs()
+
+    logger.info(s"Deleting dataset by moving it from $dataSourcePath to $targetPath...")
+
+    try {
+      val path = Files.move(
+        dataSourcePath,
+        targetPath
+      )
+      if (path == null) {
+        throw new Exception("Deleting dataset failed")
+      }
+      logger.info(s"Successfully moved dataset from $dataSourcePath to $targetPath...")
+      Fox.successful(())
+    } catch {
+      case e: Exception => Fox.failure(s"Deleting dataset failed: ${e.toString}", Full(e))
+    }
+  }
 }

webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DataSourceService.scala

@@ -53,10 +53,12 @@ class DataSourceService @Inject()(
     if (inboxCheckVerboseCounter >= 10) inboxCheckVerboseCounter = 0
   }
 
+  private def skipTrash(path: Path) = !path.toString.contains(".trash")
+
   def checkInbox(verbose: Boolean): Fox[Unit] = {
     if (verbose) logger.info(s"Scanning inbox ($dataBaseDir)...")
     for {
-      _ <- PathUtils.listDirectories(dataBaseDir) match {
+      _ <- PathUtils.listDirectories(dataBaseDir, skipTrash) match {
         case Full(dirs) =>
           for {
             _ <- Fox.successful(())

webknossos-datastore/conf/com.scalableminds.webknossos.datastore.routes

@@ -39,6 +39,7 @@ GET         /datasets/sample/:organizationName
 POST        /datasets/sample/:organizationName/:dataSetName/download                                             @com.scalableminds.webknossos.datastore.controllers.DataSourceController.fetchSampleDataSource(organizationName: String, dataSetName: String)
 POST        /datasets/:organizationName/:dataSetName                                                             @com.scalableminds.webknossos.datastore.controllers.DataSourceController.update(organizationName: String, dataSetName: String)
 GET         /datasets/:organizationName/:dataSetName                                                             @com.scalableminds.webknossos.datastore.controllers.DataSourceController.explore(organizationName: String, dataSetName: String)
+DELETE      /datasets/:organizationName/:dataSetName/deleteOnDisk                                                @com.scalableminds.webknossos.datastore.controllers.DataSourceController.deleteOnDisk(organizationName: String, dataSetName: String)
 
 # Actions
 GET         /triggers/checkInbox                                                                                 @com.scalableminds.webknossos.datastore.controllers.DataSourceController.triggerInboxCheck()