Merge branch 'master' of github.com:scalableminds/webknossos into org…

…a_owner * 'master' of github.com:scalableminds/webknossos: Loki batched insert (#6831) Fix editable text style (#6799) Fix dbtool schema scheck stderr capturing (#6830) Release 23.02.1 (#6827)
scalableminds · Feb 8, 2023 · 5dd39a2 · 5dd39a2
2 parents 788bee8 + 6d6cb36
commit 5dd39a2
Show file tree

Hide file tree

Showing 18 changed files with 208 additions and 148 deletions.
diff --git a/CHANGELOG.released.md b/CHANGELOG.released.md
@@ -7,6 +7,12 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Calendar Versioning](http://calver.org/) `0Y.0M.MICRO`.
 For upgrade instructions, please check the [migration guide](MIGRATIONS.released.md).
 
+## [23.02.1](https://github.com/scalableminds/webknossos/releases/tag/23.02.1) - 2023-02-07
+[Commits](https://github.com/scalableminds/webknossos/compare/23.02.0...23.02.1)
+
+### Fixed
+- Fixed a benign error message which briefly appeared after logging in. [#6810](https://github.com/scalableminds/webknossos/pull/6810)
+
 ## [23.02.0](https://github.com/scalableminds/webknossos/releases/tag/23.02.0) - 2023-02-01
 [Commits](https://github.com/scalableminds/webknossos/compare/23.01.0...23.02.0)
 

diff --git a/CHANGELOG.unreleased.md b/CHANGELOG.unreleased.md
@@ -8,7 +8,7 @@ and this project adheres to [Calendar Versioning](http://calver.org/) `0Y.0M.MIC
 For upgrade instructions, please check the [migration guide](MIGRATIONS.released.md).
 
 ## Unreleased
-[Commits](https://github.com/scalableminds/webknossos/compare/23.02.0...HEAD)
+[Commits](https://github.com/scalableminds/webknossos/compare/23.02.1...HEAD)
 
 ### Added
 - Remote datasets can now also be streamed from Google Cloud Storage URIs (`gs://`). [#6775](https://github.com/scalableminds/webknossos/pull/6775)
@@ -19,9 +19,9 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released
 - Added owner name to organization page. [#6811](https://github.com/scalableminds/webknossos/pull/6811)
 
 ### Fixed
-- Fixed a benign error message which briefly appeared after logging in. [#6810](https://github.com/scalableminds/webknossos/pull/6810)
 - Fixed saving allowed teams in dataset settings. [#6817](https://github.com/scalableminds/webknossos/pull/6817)
-- Fixed log streaming in Voxelytics workflow reports. [#6828](https://github.com/scalableminds/webknossos/pull/6828)
+- Fixed log streaming in Voxelytics workflow reports. [#6828](https://github.com/scalableminds/webknossos/pull/6828) [#6831](https://github.com/scalableminds/webknossos/pull/6831)
+- Fixed some layouting issues with line breaks in segment list/dataset info tab [#6799](https://github.com/scalableminds/webknossos/pull/6799)
 
 ### Removed
 

diff --git a/MIGRATIONS.released.md b/MIGRATIONS.released.md
@@ -5,6 +5,12 @@ See `MIGRATIONS.unreleased.md` for the changes which are not yet part of an offi
 This project adheres to [Calendar Versioning](http://calver.org/) `0Y.0M.MICRO`.
 User-facing changes are documented in the [changelog](CHANGELOG.released.md).
 
+## [23.02.1](https://github.com/scalableminds/webknossos/releases/tag/23.02.1) - 2023-02-07
+[Commits](https://github.com/scalableminds/webknossos/compare/23.02.0...23.02.1)
+
+### Postgres Evolutions:
+None.
+
 ## [23.02.0](https://github.com/scalableminds/webknossos/releases/tag/23.02.0) - 2023-02-01
 [Commits](https://github.com/scalableminds/webknossos/compare/23.01.0...23.02.0)
 
@@ -511,4 +517,4 @@ No migrations necessary.
 
 
 ## [18.07.0](https://github.com/scalableminds/webknossos/releases/tag/18.07.0) - 2018-07-05
-First release
+First release
diff --git a/MIGRATIONS.unreleased.md b/MIGRATIONS.unreleased.md
@@ -6,7 +6,7 @@ This project adheres to [Calendar Versioning](http://calver.org/) `0Y.0M.MICRO`.
 User-facing changes are documented in the [changelog](CHANGELOG.released.md).
 
 ## Unreleased
-[Commits](https://github.com/scalableminds/webknossos/compare/23.02.0...HEAD)
+[Commits](https://github.com/scalableminds/webknossos/compare/23.02.1...HEAD)
 
 - WEBKNOSSOS now requires Node.js not only for development and building, but also for execution. The prebuilt Docker images already contain this dependency. If you're using these, nothing needs to be changed. [#6803](https://github.com/scalableminds/webknossos/pull/6803)
 

diff --git a/app/Startup.scala b/app/Startup.scala
@@ -96,21 +96,22 @@ class Startup @Inject()(actorSystem: ActorSystem,
 
     val errorMessageBuilder = mutable.ListBuffer[String]()
     val capturingProcessLogger =
-      ProcessLogger((o: String) => errorMessageBuilder.append(o), (e: String) => logger.error(e))
+      ProcessLogger((o: String) => errorMessageBuilder.append(o), (e: String) => errorMessageBuilder.append(e))
 
     val result = Process("./tools/postgres/dbtool.js check-db-schema", None, "POSTGRES_URL" -> postgresUrl) ! capturingProcessLogger
     if (result == 0) {
       logger.info("Database schema is up to date.")
     } else {
       val errorMessage = errorMessageBuilder.toList.mkString("\n")
-      logger.error(errorMessage)
+      logger.error("dbtool: " + errorMessage)
       slackNotificationService.warn("SQL schema mismatch", errorMessage)
     }
   }
 
   private def ensurePostgresDatabase(): Unit = {
     logger.info(s"Ensuring Postgres database…")
-    val processLogger = ProcessLogger((o: String) => logger.info(o), (e: String) => logger.error(e))
+    val processLogger =
+      ProcessLogger((o: String) => logger.info(s"dbtool: $o"), (e: String) => logger.error(s"dbtool: $e"))
 
     // this script is copied to the stage directory in AssetCompilation
     val result = Process("./tools/postgres/dbtool.js ensure-db", None, "POSTGRES_URL" -> postgresUrl) ! processLogger

diff --git a/app/controllers/VoxelyticsController.scala b/app/controllers/VoxelyticsController.scala
@@ -248,7 +248,7 @@ class VoxelyticsController @Inject()(
         _ <- bool2Fox(wkConf.Features.voxelyticsEnabled) ?~> "voxelytics.disabled"
         organization <- organizationDAO.findOne(request.identity._organization)
         logEntries = request.body
-        _ <- lokiClient.bulkInsert(logEntries, organization._id)
+        _ <- lokiClient.bulkInsertBatched(logEntries, organization._id)
       } yield Ok
     }
 
@@ -257,7 +257,7 @@ class VoxelyticsController @Inject()(
               minLevel: Option[String],
               startTimestamp: Long,
               endTimestamp: Long,
-              limit: Option[Long]): Action[AnyContent] =
+              limit: Option[Int]): Action[AnyContent] =
     sil.SecuredAction.async { implicit request =>
       {
         for {

diff --git a/app/models/voxelytics/LokiClient.scala b/app/models/voxelytics/LokiClient.scala
@@ -28,7 +28,8 @@ class LokiClient @Inject()(wkConf: WkConf, rpc: RPC, val system: ActorSystem)(im
 
   private val POLLING_INTERVAL = 1 second
   private val LOG_TIME_BATCH_INTERVAL = 1 days
-  private val LOG_ENTRY_BATCH_SIZE = 5000L
+  private val LOG_ENTRY_QUERY_BATCH_SIZE = 5000
+  private val LOG_ENTRY_INSERT_BATCH_SIZE = 1000
 
   private lazy val serverStartupFuture: Fox[Unit] = {
     for {
@@ -83,7 +84,7 @@ class LokiClient @Inject()(wkConf: WkConf, rpc: RPC, val system: ActorSystem)(im
                        minLevel: VoxelyticsLogLevel = VoxelyticsLogLevel.INFO,
                        startTime: Instant,
                        endTime: Instant,
-                       limit: Option[Long]): Fox[List[JsValue]] = {
+                       limit: Option[Int]): Fox[List[JsValue]] = {
     val currentEndTime = endTime
     val currentStartTime = startTime.max(endTime - LOG_TIME_BATCH_INTERVAL)
 
@@ -94,10 +95,10 @@ class LokiClient @Inject()(wkConf: WkConf, rpc: RPC, val system: ActorSystem)(im
                              minLevel,
                              currentStartTime,
                              currentEndTime,
-                             limit.getOrElse(LOG_ENTRY_BATCH_SIZE).min(LOG_ENTRY_BATCH_SIZE))
+                             limit.getOrElse(LOG_ENTRY_QUERY_BATCH_SIZE).min(LOG_ENTRY_QUERY_BATCH_SIZE))
       newLimit = limit.map(l => (l - headBatch.length).max(0))
       buffer <- if (headBatch.isEmpty) {
-        if (currentStartTime == startTime || newLimit.contains(0L)) {
+        if (currentStartTime == startTime || newLimit.contains(0)) {
           Fox.successful(List())
         } else {
           for {
@@ -130,13 +131,13 @@ class LokiClient @Inject()(wkConf: WkConf, rpc: RPC, val system: ActorSystem)(im
     } yield buffer
   }
 
-  def queryLogs(runName: String,
-                organizationId: ObjectId,
-                taskName: Option[String],
-                minLevel: VoxelyticsLogLevel = VoxelyticsLogLevel.INFO,
-                startTime: Instant,
-                endTime: Instant,
-                limit: Long): Fox[List[JsValue]] = {
+  private def queryLogs(runName: String,
+                        organizationId: ObjectId,
+                        taskName: Option[String],
+                        minLevel: VoxelyticsLogLevel,
+                        startTime: Instant,
+                        endTime: Instant,
+                        limit: Int): Fox[List[JsValue]] = {
     val levels = VoxelyticsLogLevel.sortedValues.drop(VoxelyticsLogLevel.sortedValues.indexOf(minLevel))
 
     val logQLFilter = List(
@@ -171,7 +172,13 @@ class LokiClient @Inject()(wkConf: WkConf, rpc: RPC, val system: ActorSystem)(im
     } yield logEntries
   }
 
-  def bulkInsert(logEntries: List[JsValue], organizationId: ObjectId)(implicit ec: ExecutionContext): Fox[Unit] =
+  def bulkInsertBatched(logEntries: List[JsValue], organizationId: ObjectId)(implicit ec: ExecutionContext): Fox[Unit] =
+    for {
+      _ <- Fox.serialCombined(logEntries.grouped(LOG_ENTRY_INSERT_BATCH_SIZE).toList)(bulkInsert(_, organizationId))
+    } yield ()
+
+  private def bulkInsert(logEntries: List[JsValue], organizationId: ObjectId)(
+      implicit ec: ExecutionContext): Fox[Unit] =
     if (logEntries.nonEmpty) {
       for {
         _ <- serverStartupFuture

diff --git a/conf/webknossos.latest.routes b/conf/webknossos.latest.routes
@@ -282,4 +282,4 @@ POST          /voxelytics/workflows/:workflowHash/events
 GET           /voxelytics/workflows/:workflowHash/chunkStatistics                       controllers.VoxelyticsController.getChunkStatistics(workflowHash: String, runId: Option[String], taskName: String)
 GET           /voxelytics/workflows/:workflowHash/artifactChecksums                     controllers.VoxelyticsController.getArtifactChecksums(workflowHash: String, runId: Option[String], taskName: String, artifactName: Option[String])
 POST          /voxelytics/logs                                                          controllers.VoxelyticsController.appendLogs
-GET           /voxelytics/logs                                                          controllers.VoxelyticsController.getLogs(runId: String, taskName: Option[String], minLevel: Option[String], startTimestamp:Long, endTimestamp: Long, limit: Option[Long])
+GET           /voxelytics/logs                                                          controllers.VoxelyticsController.getLogs(runId: String, taskName: Option[String], minLevel: Option[String], startTimestamp:Long, endTimestamp: Long, limit: Option[Int])
diff --git a/frontend/javascripts/admin/dataset/dataset_upload_view.tsx b/frontend/javascripts/admin/dataset/dataset_upload_view.tsx
@@ -743,7 +743,7 @@ class DatasetUploadView extends React.Component<PropsWithFormAndRouter, State> {
                     width: 400,
                   }}
                   allowDecimals
-                  onChange={(scale) => {
+                  onChange={(scale: Vector3) => {
                     if (this.formRef.current == null) return;
                     this.formRef.current.setFieldsValue({
                       scale,

diff --git a/frontend/javascripts/components/text_with_description.tsx b/frontend/javascripts/components/text_with_description.tsx
@@ -4,11 +4,13 @@ import Markdown from "react-remarkable";
 import * as React from "react";
 import type { EditableTextLabelProp } from "oxalis/view/components/editable_text_label";
 import EditableTextLabel from "oxalis/view/components/editable_text_label";
+
 type EditableProps = EditableTextLabelProp & {
   isEditable: true;
   description: string;
 };
 type NonEditableProps = {
+  markdown?: boolean;
   isEditable: false;
   description: string;
   value: string;
@@ -61,8 +63,7 @@ class TextWithDescription extends React.PureComponent<Props> {
             </Tooltip>
           ) : null}
         </span>
-        {/* @ts-expect-error ts-migrate(2322) FIXME: Type 'string | null' is not assignable to type 'st... Remove this comment to see the full error message */}
-        <span className={hasDescription ? "flex-item" : null}>
+        <span className={hasDescription ? "flex-item" : undefined}>
           {isEditable ? (
             // @ts-expect-error ts-migrate(2769) FIXME: No overload matches this call.
             <EditableTextLabel {...editableProps} />
@@ -73,7 +74,6 @@ class TextWithDescription extends React.PureComponent<Props> {
                 display: "inline-block",
               }}
             >
-              {/* @ts-expect-error ts-migrate(2339) FIXME: Property 'markdown' does not exist on type 'Readon... Remove this comment to see the full error message */}
               {this.props.markdown ? (
                 <Markdown
                   source={this.props.value}

diff --git a/frontend/javascripts/libs/vector_input.tsx b/frontend/javascripts/libs/vector_input.tsx
@@ -2,14 +2,18 @@ import * as React from "react";
 import _ from "lodash";
 import type { ServerBoundingBoxTypeTuple } from "types/api_flow_types";
 import type { Vector3, Vector6 } from "oxalis/constants";
-import InputComponent, { InputComponentProps } from "oxalis/view/components/input_component";
+import InputComponent, {
+  InputComponentCommonProps,
+  InputElementProps,
+} from "oxalis/view/components/input_component";
 import * as Utils from "libs/utils";
 
-type BaseProps<T> = Omit<InputComponentProps, "value"> & {
+type BaseProps<T> = Omit<InputComponentCommonProps & InputElementProps, "value" | "onChange"> & {
   value: T | string;
   onChange: (value: T) => void;
   changeOnlyOnBlur?: boolean;
   allowDecimals?: boolean;
+  autoSize?: boolean;
 };
 type State = {
   isEditing: boolean;
@@ -51,7 +55,7 @@ class BaseVector<T extends Vector3 | Vector6> extends React.PureComponent<BasePr
     return value;
   }
 
-  handleBlur = () => {
+  handleBlur = (_: React.FocusEvent<HTMLInputElement>) => {
     this.setState({
       isEditing: false,
     });
@@ -89,7 +93,7 @@ class BaseVector<T extends Vector3 | Vector6> extends React.PureComponent<BasePr
     return vector;
   }
 
-  handleFocus = () => {
+  handleFocus = (_event: React.FocusEvent<HTMLInputElement>) => {
     this.setState({
       isEditing: true,
       text: this.getText(this.props.value),
@@ -127,7 +131,6 @@ class BaseVector<T extends Vector3 | Vector6> extends React.PureComponent<BasePr
 
     return (
       <InputComponent
-        // @ts-expect-error ts-migrate(2769) FIXME: No overload matches this call.
         onChange={this.handleChange}
         onFocus={this.handleFocus}
         onBlur={this.handleBlur}
@@ -136,6 +139,7 @@ class BaseVector<T extends Vector3 | Vector6> extends React.PureComponent<BasePr
           autoSize ? { ...style, width: this.getText(this.state.text).length * 8 + 25 } : style
         }
         {...props}
+        isTextArea={false}
       />
     );
   }
@@ -147,7 +151,7 @@ export class Vector3Input extends BaseVector<Vector3> {
 export class Vector6Input extends BaseVector<Vector6> {
   defaultValue: Vector6 = [0, 0, 0, 0, 0, 0];
 }
-type BoundingBoxInputProps = Omit<InputComponentProps, "value"> & {
+type BoundingBoxInputProps = Omit<InputComponentCommonProps & InputElementProps, "value"> & {
   value: ServerBoundingBoxTypeTuple;
   onChange: (arg0: ServerBoundingBoxTypeTuple) => void;
 };
@@ -179,7 +183,6 @@ export class BoundingBoxInput extends React.PureComponent<BoundingBoxInputProps>
         {...props}
         value={vector6Value}
         changeOnlyOnBlur
-        // @ts-expect-error ts-migrate(2322) FIXME: Type '([x, y, z, width, height, depth]: [any, any,... Remove this comment to see the full error message
         onChange={([x, y, z, width, height, depth]) =>
           onChange({
             topLeft: [x, y, z],

diff --git a/frontend/javascripts/oxalis/view/action-bar/dataset_position_view.tsx b/frontend/javascripts/oxalis/view/action-bar/dataset_position_view.tsx
@@ -129,7 +129,6 @@ class DatasetPositionView extends PureComponent<Props> {
           </Tooltip>
           <Vector3Input
             value={position}
-            // @ts-expect-error ts-migrate(2322) FIXME: Type '(position: Vector3) => void' is not assignab... Remove this comment to see the full error message
             onChange={this.handleChangePosition}
             autoSize
             // @ts-expect-error ts-migrate(2322) FIXME: Type '{ textAlign: string; }' is not assignable to... Remove this comment to see the full error message
@@ -159,7 +158,6 @@ class DatasetPositionView extends PureComponent<Props> {
             </Tooltip>
             <Vector3Input
               value={rotation}
-              // @ts-expect-error ts-migrate(2322) FIXME: Type '(rotation: Vector3) => void' is not assignab... Remove this comment to see the full error message
               onChange={this.handleChangeRotation}
               style={{
                 textAlign: "center",