add button to reload data buckets for each layer

CHANGELOG.md

@@ -11,6 +11,7 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.md).
 
 ### Added
 - Added `publicUri` configs for datastore and tracingstore for initial setup. [#4368](https://github.com/scalableminds/webknossos/pull/4368)
+- Added a button to delete all cached data buckets of color layer and the reload them. [#4383](https://github.com/scalableminds/webknossos/pull/4383)
 - Added a scale to the y-axis of histograms to indicate the logarithmic representation. Additionally, small histogram values are smoothed out. [#4349](https://github.com/scalableminds/webknossos/pull/4349)
 - Added a new way of sharing annotations. You can share your annotations with selected teams. These annotations appear in the Shared Annotations Tab in the dashboard. [#4304](https://github.com/scalableminds/webknossos/pull/4304)
 

frontend/javascripts/admin/admin_rest_api.js

@@ -886,10 +886,13 @@ export async function triggerDatasetCheck(datastoreHost: string): Promise<void>
 export async function triggerDatasetClearCache(
   datastoreHost: string,
   datasetId: APIDatasetId,
+  layerName?: string,
 ): Promise<void> {
   await doWithToken(token =>
     Request.triggerRequest(
-      `/data/triggers/reload/${datasetId.owningOrganization}/${datasetId.name}?token=${token}`,
+      `/data/triggers/reload/${datasetId.owningOrganization}/${datasetId.name}?token=${token}${
+        layerName ? `&layerName=${layerName}` : ""
+      }`,
       {
         host: datastoreHost,
       },
@@ -904,6 +907,13 @@ export async function triggerDatasetClearThumbnailCache(datasetId: APIDatasetId)
   );
 }
 
+export async function clearCache(dataset: APIMaybeUnimportedDataset, layerName?: string) {
+  return Promise.all([
+    triggerDatasetClearCache(dataset.dataStore.url, dataset, layerName),
+    triggerDatasetClearThumbnailCache(dataset),
+  ]);
+}
+
 export async function getDatasetSharingToken(
   datasetId: APIDatasetId,
   options?: RequestOptions,

frontend/javascripts/dashboard/advanced_dataset/dataset_action_view.js

@@ -4,11 +4,7 @@ import { Link, type RouterHistory, withRouter } from "react-router-dom";
 import * as React from "react";
 
 import type { APIMaybeUnimportedDataset, TracingType } from "admin/api_flow_types";
-import {
-  createExplorational,
-  triggerDatasetClearCache,
-  triggerDatasetClearThumbnailCache,
-} from "admin/admin_rest_api";
+import { createExplorational, clearCache } from "admin/admin_rest_api";
 import Toast from "libs/toast";
 import messages from "messages";
 import { trackAction } from "oxalis/model/helpers/analytics";
@@ -55,10 +51,7 @@ class DatasetActionView extends React.PureComponent<Props, State> {
   };
 
   clearCache = async (dataset: APIMaybeUnimportedDataset) => {
-    await Promise.all([
-      triggerDatasetClearCache(dataset.dataStore.url, dataset),
-      triggerDatasetClearThumbnailCache(dataset),
-    ]);
+    await clearCache(dataset);
     Toast.success(messages["dataset.clear_cache_success"]);
   };
 

frontend/javascripts/oxalis/api/api_latest.js

@@ -699,10 +699,25 @@ class DataApi {
   }
 
   /**
-   * Invalidates all downloaded buckets so that they are reloaded on the next movement.
+   * Invalidates all downloaded buckets of the given layer so that they are reloaded.
+   */
+  reloadBuckets(layerName: string): void {
+    _.forEach(this.model.dataLayers, dataLayer => {
+      if (dataLayer.name === layerName) {
+        dataLayer.cube.collectAllBuckets();
+        dataLayer.layerRenderingManager.refresh();
+      }
+    });
+  }
+
+  /**
+   * Invalidates all downloaded buckets so that they are reloaded.
    */
   reloadAllBuckets(): void {
-    _.forEach(this.model.dataLayers, dataLayer => dataLayer.cube.collectAllBuckets());
+    _.forEach(this.model.dataLayers, dataLayer => {
+      dataLayer.cube.collectAllBuckets();
+      dataLayer.layerRenderingManager.refresh();
+    });
   }
 
   /**

frontend/javascripts/oxalis/view/settings/dataset_settings_view.js

@@ -9,6 +9,7 @@ import { connect } from "react-redux";
 import * as React from "react";
 import _ from "lodash";
 import { V3 } from "libs/mjs";
+import api from "oxalis/api/internal_api";
 
 import type { APIDataset, APIHistogramData } from "admin/api_flow_types";
 import { AsyncIconButton } from "components/async_clickables";
@@ -18,7 +19,7 @@ import {
   DropdownSetting,
   ColorSetting,
 } from "oxalis/view/settings/setting_input_views";
-import { findDataPositionForLayer, getHistogramForLayer } from "admin/admin_rest_api";
+import { findDataPositionForLayer, getHistogramForLayer, clearCache } from "admin/admin_rest_api";
 import { getGpuFactorsWithLabels } from "oxalis/model/bucket_data_handling/data_rendering_logic";
 import { getMaxZoomValueForResolution } from "oxalis/model/accessors/flycam_accessor";
 import {
@@ -130,15 +131,37 @@ class DatasetSettings extends React.PureComponent<DatasetSettingsProps, State> {
               : () => Promise.resolve()
           }
           style={{
-            float: "right",
-            marginTop: 4,
+            position: "absolute",
+            top: 4,
+            right: -16,
             cursor: !isDisabled ? "pointer" : "not-allowed",
           }}
         />
       </Tooltip>
     );
   };
 
+  getReloadDataButton = (layerName: string) => {
+    const tooltipText =
+      "Reload the data from the server. Use this when the data on the server changed.";
+    // If the tracing contains a volume tracing, the backend can only
+    // search in the fallback layer of the segmentation layer for data.
+    return (
+      <Tooltip title={tooltipText}>
+        <AsyncIconButton
+          type="reload"
+          onClick={() => this.reloadLayerData(layerName)}
+          style={{
+            position: "absolute",
+            top: 4,
+            right: 6,
+            cursor: "pointer",
+          }}
+        />
+      </Tooltip>
+    );
+  };
+
   setVisibilityForAllLayers = (isVisible: boolean) => {
     const { layers } = this.props.datasetConfiguration;
     Object.keys(layers).forEach(otherLayerName =>
@@ -228,6 +251,7 @@ class DatasetSettings extends React.PureComponent<DatasetSettingsProps, State> {
           <span style={{ fontWeight: 700 }}>{layerName}</span>
           <Tag style={{ cursor: "default", marginLeft: 8 }}>{elementClass} Layer</Tag>
           {this.getFindDataButton(layerName, isDisabled, isColorLayer)}
+          {this.getReloadDataButton(layerName)}
         </Col>
       </Row>
     );
@@ -329,6 +353,15 @@ class DatasetSettings extends React.PureComponent<DatasetSettingsProps, State> {
     );
   };
 
+  reloadLayerData = async (layerName: string): Promise<void> => {
+    await clearCache(this.props.dataset, layerName);
+    api.data.reloadBuckets(layerName);
+    window.needsRerender = true;
+    Toast.success(
+      `Successfully deleted cached data of layer ${layerName}. Now move a bit around to reload the data.`,
+    );
+  };
+
   onChangeRenderMissingDataBlack = (value: boolean): void => {
     Toast.warning(
       value

util/src/main/scala/com/scalableminds/util/cache/LRUConcurrentCache.scala

@@ -43,9 +43,7 @@ trait LRUConcurrentCache[K, V] {
     cache.synchronized {
       val matching = cache.keySet.asScala.filter(predicate)
       val size = matching.size
-      matching.map { key =>
-        remove(key)
-      }
+      matching.foreach(remove)
       size
     }
 

webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DataSourceController.scala

@@ -191,19 +191,21 @@ class DataSourceController @Inject()(
     }
   }
 
-  def reload(organizationName: String, dataSetName: String) = Action.async { implicit request =>
-    accessTokenService.validateAccess(UserAccessRequest.administrateDataSources) {
-      AllowRemoteOrigin {
-        val count = binaryDataServiceHolder.binaryDataService.clearCache(organizationName, dataSetName)
-        logger.info(s"Reloading datasource $organizationName / $dataSetName : closed " + count + " open file handles.")
-        val reloadedDataSource = dataSourceService.dataSourceFromFolder(
-          dataSourceService.dataBaseDir.resolve(organizationName).resolve(dataSetName),
-          organizationName)
-        for {
-          _ <- dataSourceRepository.updateDataSource(reloadedDataSource)
-        } yield Ok(Json.toJson(reloadedDataSource))
+  def reload(organizationName: String, dataSetName: String, layerName: Option[String] = None) = Action.async {
+    implicit request =>
+      accessTokenService.validateAccess(UserAccessRequest.administrateDataSources) {
+        AllowRemoteOrigin {
+          val count = binaryDataServiceHolder.binaryDataService.clearCache(organizationName, dataSetName, layerName)
+          logger.info(
+            s"Reloading ${layerName.map(l => s"layer '$l' of ").getOrElse("")}datasource $organizationName / $dataSetName: closed $count open file handles.")
+          val reloadedDataSource = dataSourceService.dataSourceFromFolder(
+            dataSourceService.dataBaseDir.resolve(organizationName).resolve(dataSetName),
+            organizationName)
+          for {
+            _ <- dataSourceRepository.updateDataSource(reloadedDataSource)
+          } yield Ok(Json.toJson(reloadedDataSource))
+        }
       }
-    }
   }
 
 }

webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/BinaryDataService.scala

@@ -168,9 +168,10 @@ class BinaryDataService(dataBaseDir: Path, loadTimeout: FiniteDuration, maxCache
     result
   }
 
-  def clearCache(organizationName: String, dataSetName: String) = {
+  def clearCache(organizationName: String, dataSetName: String, layerName: Option[String]) = {
     def matchingPredicate(cubeKey: CachedCube) =
-      cubeKey.dataSourceName == dataSetName
+      cubeKey.dataSourceName == dataSetName && cubeKey.organization == organizationName && layerName.forall(
+        _ == cubeKey.dataLayerName)
 
     cache.clear(matchingPredicate)
   }

webknossos-datastore/conf/com.scalableminds.webknossos.datastore.routes

@@ -41,4 +41,4 @@ GET         /datasets/:organizationName/:dataSetName
 GET         /triggers/checkInbox                                                                                 @com.scalableminds.webknossos.datastore.controllers.DataSourceController.triggerInboxCheck()
 GET         /triggers/checkInboxBlocking                                                                         @com.scalableminds.webknossos.datastore.controllers.DataSourceController.triggerInboxCheckBlocking()
 GET         /triggers/newOrganizationFolder                                                                      @com.scalableminds.webknossos.datastore.controllers.DataSourceController.createOrganizationDirectory(organizationName: String)
-GET         /triggers/reload/:organizationName/:dataSetName                                                      @com.scalableminds.webknossos.datastore.controllers.DataSourceController.reload(organizationName: String, dataSetName: String)
+GET         /triggers/reload/:organizationName/:dataSetName                                                      @com.scalableminds.webknossos.datastore.controllers.DataSourceController.reload(organizationName: String, dataSetName: String, layerName: Option[String])