Merge branch 'master' into delete-workflow-reports

scalableminds · Oct 30, 2024 · fd64a57 · fd64a57
2 parents 5ad996b + d40dfbe
commit fd64a57
Show file tree

Hide file tree

Showing 31 changed files with 245 additions and 59 deletions.
diff --git a/CHANGELOG.unreleased.md b/CHANGELOG.unreleased.md
@@ -16,6 +16,7 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released
 - Most sliders have been improved: Wheeling above a slider now changes its value and double-clicking its knob resets it to its default value. [#8095](https://github.com/scalableminds/webknossos/pull/8095)
 - It is now possible to search for unnamed segments with the full default name instead of only their id. [#8133](https://github.com/scalableminds/webknossos/pull/8133)
 - Increased loading speed for precomputed meshes. [#8110](https://github.com/scalableminds/webknossos/pull/8110)
+- Added a button to the search popover in the skeleton and segment tab to select all matching non-group results. [#8123](https://github.com/scalableminds/webknossos/pull/8123)
 - Unified wording in UI and code: “Magnification”/“mag” is now used in place of “Resolution“ most of the time, compare [https://docs.webknossos.org/webknossos/terminology.html](terminology document). [#8111](https://github.com/scalableminds/webknossos/pull/8111)
 - Added support for adding remote OME-Zarr NGFF version 0.5 datasets. [#8122](https://github.com/scalableminds/webknossos/pull/8122)
 - Workflow reports may be deleted. [#8156](https://github.com/scalableminds/webknossos/pull/8156)
@@ -28,6 +29,7 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released
 
 ### Fixed
 - Fixed a bug during dataset upload in case the configured `datastore.baseFolder` is an absolute path. [#8098](https://github.com/scalableminds/webknossos/pull/8098) [#8103](https://github.com/scalableminds/webknossos/pull/8103)
+- Fixed bbox export menu item [#8152](https://github.com/scalableminds/webknossos/pull/8152)
 - When trying to save an annotation opened via a link including a sharing token, the token is automatically discarded in case it is insufficient for update actions but the users token is. [#8139](https://github.com/scalableminds/webknossos/pull/8139)
 - Fixed that the skeleton search did not automatically expand groups that contained the selected tree [#8129](https://github.com/scalableminds/webknossos/pull/8129)
 - Fixed a bug that zarr streaming version 3 returned the shape of mag (1, 1, 1) / the finest mag for all mags. [#8116](https://github.com/scalableminds/webknossos/pull/8116)

diff --git a/MIGRATIONS.unreleased.md b/MIGRATIONS.unreleased.md
@@ -12,3 +12,4 @@ User-facing changes are documented in the [changelog](CHANGELOG.released.md).
 
 - [121-worker-name.sql](conf/evolutions/121-worker-name.sql)
 - [122-resolution-to-mag.sql](conf/evolutions/122-resolution-to-mag.sql)
+- [123-more-model-categories.sql](conf/evolutions/123-more-model-categories.sql)
diff --git a/app/controllers/AiModelController.scala b/app/controllers/AiModelController.scala
@@ -57,6 +57,16 @@ object UpdateAiModelParameters {
   implicit val jsonFormat: OFormat[UpdateAiModelParameters] = Json.format[UpdateAiModelParameters]
 }
 
+case class RegisterAiModelParameters(id: ObjectId, // must be a valid MongoDB ObjectId
+                                     dataStoreName: String,
+                                     name: String,
+                                     comment: Option[String],
+                                     category: Option[AiModelCategory])
+
+object RegisterAiModelParameters {
+  implicit val jsonFormat: OFormat[RegisterAiModelParameters] = Json.format[RegisterAiModelParameters]
+}
+
 class AiModelController @Inject()(
     aiModelDAO: AiModelDAO,
     aiModelService: AiModelService,
@@ -209,6 +219,28 @@ class AiModelController @Inject()(
       } yield Ok(jsResult)
     }
 
+  def registerAiModel: Action[RegisterAiModelParameters] =
+    sil.SecuredAction.async(validateJson[RegisterAiModelParameters]) { implicit request =>
+      for {
+        _ <- userService.assertIsSuperUser(request.identity)
+        _ <- dataStoreDAO.findOneByName(request.body.dataStoreName) ?~> "dataStore.notFound"
+        _ <- aiModelDAO.findOne(request.body.id).reverse ?~> "aiModel.id.taken"
+        _ <- aiModelDAO.findOneByName(request.body.name).reverse ?~> "aiModel.name.taken"
+        _ <- aiModelDAO.insertOne(
+          AiModel(
+            request.body.id,
+            _organization = request.identity._organization,
+            request.body.dataStoreName,
+            request.identity._id,
+            None,
+            List.empty,
+            request.body.name,
+            request.body.comment,
+            request.body.category
+          ))
+      } yield Ok
+    }
+
   def deleteAiModel(aiModelId: String): Action[AnyContent] =
     sil.SecuredAction.async { implicit request =>
       for {

diff --git a/app/models/aimodels/AiModel.scala b/app/models/aimodels/AiModel.scala
@@ -144,4 +144,11 @@ class AiModelDAO @Inject()(sqlClient: SqlClient)(implicit ec: ExecutionContext)
         q"UPDATE webknossos.aiModels SET name = ${a.name}, comment = ${a.comment}, modified = ${a.modified} WHERE _id = ${a._id}".asUpdate)
     } yield ()
 
+  def findOneByName(name: String)(implicit ctx: DBAccessContext): Fox[AiModel] =
+    for {
+      accessQuery <- readAccessQuery
+      r <- run(q"SELECT $columns FROM $existingCollectionName WHERE name = $name AND $accessQuery".as[AimodelsRow])
+      parsed <- parseFirst(r, name)
+    } yield parsed
+
 }
diff --git a/app/models/aimodels/AiModelCategory.scala b/app/models/aimodels/AiModelCategory.scala
@@ -4,5 +4,5 @@ import com.scalableminds.util.enumeration.ExtendedEnumeration
 
 object AiModelCategory extends ExtendedEnumeration {
   type AiModelCategory = Value
-  val em_neurons, em_nuclei = Value
+  val em_neurons, em_nuclei, em_synapses, em_neuron_types, em_cell_organelles = Value
 }
diff --git a/app/utils/sql/SQLDAO.scala b/app/utils/sql/SQLDAO.scala
@@ -47,7 +47,7 @@ abstract class SQLDAO[C, R, X <: AbstractTable[R]] @Inject()(sqlClient: SqlClien
       case Some(r) =>
         parse(r) ?~> ("sql: could not parse database row for object" + id)
       case _ =>
-        Fox.failure("sql: could not find object " + id)
+        Fox.empty
     }.flatten
 
   @nowarn // suppress warning about unused implicit ctx, as it is used in subclasses

diff --git a/app/views/mail/jobFailedUploadConvert.scala.html b/app/views/mail/jobFailedUploadConvert.scala.html
@@ -11,9 +11,9 @@
 
 <p>Here are some tips for uploading and converting datasets:
 <ul>
-  <li><a href="https://docs.webknossos.org/webknossos/data_formats.html">See the document on supported files formats</a></li>
+  <li><a href="https://docs.webknossos.org/webknossos/data/index.html">See the document on supported files formats</a></li>
   <li><a href="https://docs.webknossos.org/webknossos-py/index.html">Try our Python library for uploading datasets</a></li>
-  <li><a href="https://docs.webknossos.org/webknossos/datasets.html#working-with-zarr-neuroglancer-precomputed-and-n5-datasets">Try streaming your data as Zarr, Neuroglancer, or N5 files instead of uploading</a></li>
+  <li><a href="https://docs.webknossos.org/webknossos/data/streaming.html">Try streaming your data as Zarr, Neuroglancer, or N5 files instead of uploading</a></li>
 </ul>
 </p>
 

diff --git a/app/views/mail/jobSuccessfulSegmentation.scala.html b/app/views/mail/jobSuccessfulSegmentation.scala.html
@@ -38,7 +38,7 @@
 </div>
 
 <p>
-  Do you want to make corrections to the automated segmentation? Use the easy-to-use, built-in <a href="https://docs.webknossos.org/webknossos/proof_reading.html#proofreading-tool">proof-reading tools in WEBKNOSSOS</a> (requires Power plan).
+  Do you want to make corrections to the automated segmentation? Use the easy-to-use, built-in <a href="https://docs.webknossos.org/webknossos/proofreading/tools.html">proof-reading tools in WEBKNOSSOS</a> (requires Power plan).
 </p>
 <div style="text-align: center; margin-bottom: 20px;">
   <img src="https://static.webknossos.org/mails/email-proofreading-preview.600.jpg" 

diff --git a/conf/evolutions/123-more-model-categories.sql b/conf/evolutions/123-more-model-categories.sql
@@ -0,0 +1,11 @@
+
+-- no transaction here, since ALTER TYPE ... ADD cannot run inside a transaction block
+
+do $$ begin ASSERT (select schemaVersion from webknossos.releaseInformation) = 122, 'Previous schema version mismatch'; end; $$ LANGUAGE plpgsql;
+
+ALTER TYPE webknossos.AI_MODEL_CATEGORY ADD VALUE 'em_synapses';
+ALTER TYPE webknossos.AI_MODEL_CATEGORY ADD VALUE 'em_neuron_types';
+ALTER TYPE webknossos.AI_MODEL_CATEGORY ADD VALUE 'em_cell_organelles';
+
+UPDATE webknossos.releaseInformation SET schemaVersion = 123;
+
diff --git a/conf/evolutions/reversions/123-more-model-categories.sql b/conf/evolutions/reversions/123-more-model-categories.sql
@@ -0,0 +1,11 @@
+START TRANSACTION;
+
+do $$ begin ASSERT (select schemaVersion from webknossos.releaseInformation) = 123, 'Previous schema version mismatch'; end; $$ LANGUAGE plpgsql;
+
+-- removing enum values is not supported in postgres, see https://www.postgresql.org/docs/current/datatype-enum.html#DATATYPE-ENUM-IMPLEMENTATION-DETAILS
+
+UPDATE webknossos.aiModels SET isDeleted = TRUE WHERE category IN ('em_synapses', 'em_neuron_types', 'em_cell_organelles');
+
+UPDATE webknossos.releaseInformation SET schemaVersion = 122;
+
+COMMIT TRANSACTION;
diff --git a/conf/webknossos.latest.routes b/conf/webknossos.latest.routes
@@ -283,6 +283,7 @@ POST          /aiModels/inferences/runInference
 GET           /aiModels/inferences/:id                                                          controllers.AiModelController.readAiInferenceInfo(id: String)
 GET           /aiModels/inferences                                                              controllers.AiModelController.listAiInferences
 GET           /aiModels                                                                         controllers.AiModelController.listAiModels
+POST          /aiModels/register                                                                controllers.AiModelController.registerAiModel
 GET           /aiModels/:id                                                                     controllers.AiModelController.readAiModelInfo(id: String)
 PUT           /aiModels/:id                                                                     controllers.AiModelController.updateAiModelInfo(id: String)
 DELETE        /aiModels/:id                                                                     controllers.AiModelController.deleteAiModel(id: String)

diff --git a/docs/data/zarr.md b/docs/data/zarr.md
@@ -111,22 +111,27 @@ For OME-Zarr (v0.5) datasets, the structure is slightly different (See [OME-Zarr
 ## Conversion to Zarr
 
 You can easily convert image stacks manually with the [WEBKNOSSOS CLI](https://docs.webknossos.org/cli).
-The CLI tool expects all image files in a single folder with numbered file names.
+The CLI tool expects a single file or all image files in a single folder with numbered file names.
 After installing, you can convert image stacks to Zarr datasets with the following command:
 
 ```shell
-pip install webknossos
+pip install --extra-index-url https://pypi.scm.io/simple "webknossos[all]"
 
 webknossos convert \
+  --layer-name em \
   --voxel-size 11.24,11.24,25 \
-  --name my_dataset \
+  --chunk-shape 64,64,64 \
   --data-format zarr \
-  data/source data/target
+  --jobs 4 \
+  input.tif output.zarr
+
+webknossos compress --jobs 4 output.zarr
+webknossos downsample --jobs 4 output.zarr
 ```
 
-This snippet converts an image stack that is located in directory called `data/source` into a Zarr dataset which will be located at `data/target`.
-It will create a so called `color` layer containing your raw greyscale/color image.
-The supplied `--voxel-size` is specified in nanometers.
+This example will create an unsharded Zarr v2 dataset with a voxel size of (4,4,4) nm<sup>3</sup> and a chunk size of (64,64,64) voxel. 
+A maximum of 4 parallel jobs will be used to parallelize the conversion, compression and downsampling.
+Using the `--data-format zarr3` argument will produce sharded Zarr v3 datasets.
 
 Read the full documentation at [WEBKNOSSOS CLI](https://docs.webknossos.org/cli).
 
@@ -170,3 +175,5 @@ To get the best streaming performance for Zarr datasets consider the following s
 
 - Use chunk sizes of 32 - 128 voxels^3
 - Enable sharding (only available in Zarr 3+)
+- Use 3D downsampling
+
diff --git a/frontend/javascripts/admin/auth/auth_token_view.tsx b/frontend/javascripts/admin/auth/auth_token_view.tsx
@@ -102,14 +102,12 @@ function AuthTokenView() {
         <Col span={8}>
           <p>
             An Auth Token is a series of symbols that serves to authenticate you. It is used in
-            communication with the backend API and sent with every request to verify your identity.
+            communication with the Python API and sent with every request to verify your identity.
           </p>
           <p>
             You should revoke it if somebody else has acquired your token or you have the suspicion
             this has happened.{" "}
-            <a href="https://docs.webknossos.org/webknossos/rest_api.html#authentication">
-              Read more
-            </a>
+            <a href="https://docs.webknossos.org/webknossos-py/index.html">Read more</a>
           </p>
         </Col>
       </Row>

diff --git a/frontend/javascripts/admin/dataset/dataset_add_view.tsx b/frontend/javascripts/admin/dataset/dataset_add_view.tsx
@@ -183,7 +183,7 @@ const alignBanner = (
     />
     <p>
       <a
-        href="https://docs.webknossos.org/webknossos/automated_analysis.html"
+        href="https://docs.webknossos.org/webknossos/automation/alignment.html"
         target="_blank"
         rel="noopener noreferrer"
       >

diff --git a/frontend/javascripts/admin/dataset/dataset_upload_view.tsx b/frontend/javascripts/admin/dataset/dataset_upload_view.tsx
@@ -1243,7 +1243,7 @@ function FileUploadArea({
                     <Popover
                       content={
                         <a
-                          href="https://docs.webknossos.org/webknossos/zarr.html"
+                          href="https://docs.webknossos.org/webknossos/data/zarr.html"
                           target="_blank"
                           rel="noreferrer"
                           onClick={(e) => e.stopPropagation()}
@@ -1265,7 +1265,7 @@ function FileUploadArea({
                     <Popover
                       content={
                         <a
-                          href="https://docs.webknossos.org/webknossos/neuroglancer_precomputed.html"
+                          href="https://docs.webknossos.org/webknossos/data/neuroglancer_precomputed.html"
                           target="_blank"
                           rel="noreferrer"
                           onClick={(e) => e.stopPropagation()}
@@ -1287,7 +1287,7 @@ function FileUploadArea({
                     <Popover
                       content={
                         <a
-                          href="https://docs.webknossos.org/webknossos/n5.html"
+                          href="https://docs.webknossos.org/webknossos/data/n5.html"
                           target="_blank"
                           rel="noreferrer"
                           onClick={(e) => e.stopPropagation()}
@@ -1309,7 +1309,7 @@ function FileUploadArea({
                 </ul>
                 Have a look at{" "}
                 <a
-                  href="https://docs.webknossos.org/webknossos/data_formats.html"
+                  href="https://docs.webknossos.org/webknossos/data/image_stacks.html"
                   onClick={(e) => e.stopPropagation()}
                 >
                   our documentation

diff --git a/frontend/javascripts/admin/job/job_list_view.tsx b/frontend/javascripts/admin/job/job_list_view.tsx
@@ -366,7 +366,7 @@ function JobListView() {
         Some actions such as dataset conversions or export as Tiff files require some time for
         processing in the background.
         <a
-          href="https://docs.webknossos.org/webknossos/jobs.html"
+          href="https://docs.webknossos.org/webknossos/automation/jobs.html"
           target="_blank"
           rel="noopener noreferrer"
         >

diff --git a/frontend/javascripts/admin/onboarding.tsx b/frontend/javascripts/admin/onboarding.tsx
@@ -548,7 +548,7 @@ class OnboardingView extends React.PureComponent<Props, State> {
           height={250}
         >
           You can also copy it directly onto the hosting server.{" "}
-          <a href="https://docs.webknossos.org/webknossos/data_formats.html">
+          <a href="https://docs.webknossos.org/webknossos/data/index.html">
             Learn more about supported data formats.
           </a>
         </OptionCard>
@@ -583,8 +583,8 @@ class OnboardingView extends React.PureComponent<Props, State> {
         </FeatureCard>
         <FeatureCard header="More Datasets" icon={<CloudUploadOutlined />}>
           <a href="/datasets/upload">Upload more of your datasets.</a>{" "}
-          <a href="https://docs.webknossos.org/webknossos/data_formats.html">Learn more</a> about
-          the formats and upload processes WEBKNOSSOS supports.
+          <a href="https://docs.webknossos.org/webknossos/data/index.html">Learn more</a> about the
+          formats and upload processes WEBKNOSSOS supports.
         </FeatureCard>
         <FeatureCard header="User & Team Management" icon={<TeamOutlined />}>
           <LinkButton

diff --git a/frontend/javascripts/admin/statistic/available_tasks_report_view.tsx b/frontend/javascripts/admin/statistic/available_tasks_report_view.tsx
@@ -45,7 +45,7 @@ function AvailableTasksReportView() {
         available to multiple users here, but each will only be handed to the first user to request
         it.
         <a
-          href="https://docs.webknossos.org/webknossos/tasks.html"
+          href="https://docs.webknossos.org/webknossos/tasks_projects/index.html"
           target="_blank"
           rel="noopener noreferrer"
         >

diff --git a/frontend/javascripts/admin/task/task_list_view.tsx b/frontend/javascripts/admin/task/task_list_view.tsx
@@ -222,7 +222,7 @@ function TaskListView({ initialFieldValues }: Props) {
         <p>
           To learn more about the task system in WEBKNOSSOS,{" "}
           <a
-            href="https://docs.webknossos.org/webknossos/tasks.html"
+            href="https://docs.webknossos.org/webknossos/tasks_projects/index.html"
             rel="noopener noreferrer"
             target="_blank"
           >

diff --git a/frontend/javascripts/admin/user/permissions_and_teams_modal_view.tsx b/frontend/javascripts/admin/user/permissions_and_teams_modal_view.tsx
@@ -243,7 +243,7 @@ function PermissionsAndTeamsModalView({
         <h4>
           Organization Permissions{" "}
           <a
-            href="https://docs.webknossos.org/webknossos/users.html"
+            href="https://docs.webknossos.org/webknossos/users/index.html"
             target="_blank"
             rel="noopener noreferrer"
           >

diff --git a/frontend/javascripts/dashboard/dashboard_task_list_view.tsx b/frontend/javascripts/dashboard/dashboard_task_list_view.tsx
@@ -414,7 +414,7 @@ class DashboardTaskListView extends React.PureComponent<Props, State> {
                         as part of the WEBKNOSSOS project management.{" "}
                       </p>
                       <a
-                        href="https://docs.webknossos.org/webknossos/tasks.html"
+                        href="https://docs.webknossos.org/webknossos/tasks_projects/index.html"
                         rel="noopener noreferrer"
                         target="_blank"
                       >

diff --git a/frontend/javascripts/dashboard/dataset_folder_view.tsx b/frontend/javascripts/dashboard/dataset_folder_view.tsx
@@ -134,7 +134,7 @@ function DatasetFolderViewInner(props: Props) {
                 <p>
                   WEBKNOSSOS supports a variety of (remote){" "}
                   <a
-                    href="https://docs.webknossos.org/webknossos/data_formats.html"
+                    href="https://docs.webknossos.org/webknossos/data/index.html"
                     target="_blank"
                     rel="noreferrer"
                   >

diff --git a/frontend/javascripts/oxalis/view/action-bar/download_modal_view.tsx b/frontend/javascripts/oxalis/view/action-bar/download_modal_view.tsx
@@ -433,7 +433,7 @@ function _DownloadModalView({
     >
       For more information on how to work with {typeDependentFileName} visit the{" "}
       <a
-        href="https://docs.webknossos.org/webknossos/tooling.html"
+        href="https://docs.webknossos.org/webknossos/data/export_ui.html"
         target="_blank"
         rel="noreferrer"
       >
Original file line number	Diff line number	Diff line change
Expand Up		@@ -12,3 +12,4 @@ User-facing changes are documented in the [changelog](CHANGELOG.released.md).

		- [121-worker-name.sql](conf/evolutions/121-worker-name.sql)
		- [122-resolution-to-mag.sql](conf/evolutions/122-resolution-to-mag.sql)
		- [123-more-model-categories.sql](conf/evolutions/123-more-model-categories.sql)