Add manual pass state for long-running jobs

CHANGELOG.unreleased.md

@@ -11,7 +11,7 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released
 [Commits](https://github.com/scalableminds/webknossos/compare/21.06.0...HEAD)
 
 ### Added
--
+- Added the possibility for admins to set long-running jobs to a “manually repaired” state. [#5530](https://github.com/scalableminds/webknossos/pull/5530)
 
 ### Changed
 -
@@ -20,7 +20,7 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released
 -
 
 ### Removed
-- 
+-
 
 ### Breaking Change
 -

MIGRATIONS.unreleased.md

@@ -9,4 +9,4 @@ User-facing changes are documented in the [changelog](CHANGELOG.released.md).
 -
 
 ### Postgres Evolutions:
-- 
+- [072-jobs-manually-repaired.sql](conf/evolutions/072-jobs-manually-repaired.sql)

app/models/job/Job.scala

@@ -11,6 +11,7 @@ import com.scalableminds.webknossos.schema.Tables.Jobs
 import com.typesafe.scalalogging.LazyLogging
 import javax.inject.Inject
 import models.analytics.{AnalyticsService, FailedJobEvent, RunJobEvent}
+import models.job.JobManualState.JobManualState
 import models.user.{MultiUserDAO, User, UserDAO}
 import net.liftweb.common.{Failure, Full}
 import oxalis.telemetry.SlackNotificationService
@@ -30,6 +31,7 @@ case class Job(
     commandArgs: JsObject = Json.obj(),
     celeryJobId: String,
     celeryInfo: JsObject = Json.obj(),
+    manualState: Option[JobManualState] = None,
     created: Long = System.currentTimeMillis(),
     isDeleted: Boolean = false
 )
@@ -42,18 +44,21 @@ class JobDAO @Inject()(sqlClient: SQLClient)(implicit ec: ExecutionContext)
   def isDeletedColumn(x: Jobs): Rep[Boolean] = x.isdeleted
 
   def parse(r: JobsRow): Fox[Job] =
-    Fox.successful(
+    for {
+      manualStateOpt <- Fox.runOptional(r.manualstate)(JobManualState.fromString)
+    } yield {
       Job(
         ObjectId(r._Id),
         ObjectId(r._Owner),
         r.command,
         Json.parse(r.commandargs).as[JsObject],
         r.celeryjobid,
         Json.parse(r.celeryinfo).as[JsObject],
+        manualStateOpt,
         r.created.getTime,
         r.isdeleted
       )
-    )
+    }
 
   override def readAccessQ(requestingUserId: ObjectId) =
     s"""_owner = '$requestingUserId'"""
@@ -85,9 +90,10 @@ class JobDAO @Inject()(sqlClient: SQLClient)(implicit ec: ExecutionContext)
   def insertOne(j: Job): Fox[Unit] =
     for {
       _ <- run(
-        sqlu"""insert into webknossos.jobs(_id, _owner, command, commandArgs, celeryJobId, celeryInfo, created, isDeleted)
+        sqlu"""insert into webknossos.jobs(_id, _owner, command, commandArgs, celeryJobId, celeryInfo, manualState, created, isDeleted)
                          values(${j._id}, ${j._owner}, ${j.command}, '#${sanitize(j.commandArgs.toString)}', ${j.celeryJobId}, '#${sanitize(
-          j.celeryInfo.toString)}', ${new java.sql.Timestamp(j.created)}, ${j.isDeleted})""")
+          j.celeryInfo.toString)}', #${optionLiteral(j.manualState.map(s => sanitize(s.toString)))}, ${new java.sql.Timestamp(
+          j.created)}, ${j.isDeleted})""")
     } yield ()
 
   def updateCeleryInfoByCeleryId(celeryJobId: String, celeryInfo: JsObject): Fox[Unit] =
@@ -227,7 +233,8 @@ class JobService @Inject()(wkConf: WkConf,
         "commandArgs" -> job.commandArgs,
         "celeryJobId" -> job.celeryJobId,
         "created" -> job.created,
-        "celeryInfo" -> job.celeryInfo
+        "celeryInfo" -> job.celeryInfo,
+        "manualState" -> job.manualState
       ))
 
   def getCeleryInfo(job: Job): Fox[JsObject] =

app/models/job/JobManualState.scala

@@ -0,0 +1,8 @@
+package models.job
+
+import com.scalableminds.util.enumeration.ExtendedEnumeration
+
+object JobManualState extends ExtendedEnumeration {
+  type JobManualState = Value
+  val SUCCESS, FAILURE = Value
+}

conf/evolutions/072-jobs-manually-repaired.sql

@@ -0,0 +1,17 @@
+-- https://github.com/scalableminds/webknossos/pull/5530
+
+START TRANSACTION;
+
+DROP VIEW webknossos.jobs_;
+
+CREATE TYPE webknossos.JOB_MANUAL_STATE AS ENUM ('SUCCESS', 'FAILURE');
+
+ALTER TABLE webknossos.jobs ADD COLUMN manualState webknossos.JOB_MANUAL_STATE;
+
+UPDATE webknossos.jobs SET manualstate = 'FAILURE' WHERE celeryinfo->'state' = '"FAILURE"'::jsonb;
+
+CREATE VIEW webknossos.jobs_ AS SELECT * FROM webknossos.jobs WHERE NOT isDeleted;
+
+UPDATE webknossos.releaseInformation SET schemaVersion = 72;
+
+COMMIT TRANSACTION;

conf/evolutions/reversions/072-jobs-manually-repaired.sql

@@ -0,0 +1,13 @@
+START TRANSACTION;
+
+DROP VIEW webknossos.jobs_;
+
+ALTER TABLE webknossos.jobs DROP COLUMN manualState;
+
+DROP TYPE webknossos.JOB_MANUAL_STATE;
+
+CREATE VIEW webknossos.jobs_ AS SELECT * FROM webknossos.jobs WHERE NOT isDeleted;
+
+UPDATE webknossos.releaseInformation SET schemaVersion = 71;
+
+COMMIT TRANSACTION;

frontend/javascripts/admin/admin_rest_api.js

@@ -16,6 +16,9 @@ import {
   type APIFeatureToggles,
   type APIHistogramData,
   type APIJob,
+  type APIJobCeleryState,
+  type APIJobManualState,
+  type APIJobState,
   type APIMaybeUnimportedDataset,
   type APIOpenTasksReport,
   type APIOrganization,
@@ -833,11 +836,29 @@ export async function getJobs(): Promise<Array<APIJob>> {
     tracingId: job.commandArgs.kwargs.volume_tracing_id,
     annotationId: job.commandArgs.kwargs.annotation_id,
     annotationType: job.commandArgs.kwargs.annotation_type,
-    state: job.celeryInfo.state || "UNKNOWN",
+    state: adaptJobState(job.command, job.celeryInfo.state, job.manualState),
+    manualState: job.manualState,
     createdAt: job.created,
   }));
 }
 
+function adaptJobState(
+  command: string,
+  celeryState: APIJobCeleryState,
+  manualState: APIJobManualState,
+): APIJobState {
+  if (manualState) {
+    return manualState;
+  } else if (celeryState === "FAILURE" && isManualPassJobType(command)) {
+    return "MANUAL";
+  }
+  return celeryState || "UNKNOWN";
+}
+
+function isManualPassJobType(command: string) {
+  return ["convert_to_wkw"].includes(command);
+}
+
 export async function startConvertToWkwJob(
   datasetName: string,
   organizationName: string,

frontend/javascripts/admin/dataset/dataset_upload_view.js

@@ -250,15 +250,7 @@ class DatasetUploadView extends React.Component<PropsWithFormAndRouter, State> {
                   <React.Fragment>
                     The conversion for the uploaded dataset was started.
                     <br />
-                    Click{" "}
-                    <a
-                      target="_blank"
-                      href="https://github.com/scalableminds/webknossos-cuber/"
-                      rel="noopener noreferrer"
-                    >
-                      here
-                    </a>{" "}
-                    to see all running jobs.
+                    Click <a href="/jobs">here</a> to see all running jobs.
                   </React.Fragment>,
                 );
               } else {

frontend/javascripts/admin/job/job_list_view.js

@@ -140,15 +140,21 @@ class JobListView extends React.PureComponent<Props, State> {
   };
 
   renderState = (__: any, job: APIJob) => {
-    const stateString = _.capitalize(job.state.toLowerCase());
-    if (job.state === "SUCCESS") return stateString;
-    else {
-      return (
-        <Tooltip title="Something went wrong when executing this job. Feel free to contact us if you need assistance.">
-          {stateString}
-        </Tooltip>
-      );
-    }
+    const tooltipMessages = {
+      UNKNOWN:
+        "The status information for this job could not be retreived. Please try again in a few minutes, or contact us if you need assistance.",
+      SUCCESS: "This job has successfully been executed.",
+      PENDING: "This job will run as soon as a worker becomes available.",
+      STARTED: "This job is currently running.",
+      FAILURE:
+        "Something went wrong when executing this job. Feel free to contact us if you need assistance.",
+      MANUAL:
+        "The job will be handled by an admin shortly, since it couldn't be finished automatically. Please check back here soon.",
+    };
+
+    const tooltip: string = tooltipMessages[job.state];
+    const jobStateNormalized = _.capitalize(job.state.toLowerCase());
+    return <Tooltip title={tooltip}>{jobStateNormalized}</Tooltip>;
   };
 
   render() {

frontend/javascripts/types/api_flow_types.js

@@ -550,6 +550,10 @@ export type APIFeatureToggles = {
   +exportTiffMaxEdgeLengthVx: number,
 };
 
+export type APIJobCeleryState = "SUCCESS" | "PENDING" | "STARTED" | "FAILURE" | null;
+export type APIJobManualState = "SUCCESS" | "FAILURE" | null;
+export type APIJobState = "UNKNOWN" | "SUCCESS" | "PENDING" | "STARTED" | "FAILURE" | "MANUAL";
+
 export type APIJob = {
   +id: string,
   +datasetName: ?string,
@@ -562,6 +566,7 @@ export type APIJob = {
   +boundingBox: ?string,
   +type: string,
   +state: string,
+  +manualState: string,
   +createdAt: number,
 };
 

package.json

@@ -119,6 +119,8 @@
     "flow-coverage": "node_modules/flow-coverage-report/bin/flow-coverage-report.js -i \"./frontend/javascripts/**/*.js\" -x \"./frontend/javascripts/test/**/*.js\" -t html -t text",
     "docs": "node_modules/.bin/documentation build --shallow frontend/javascripts/oxalis/api/api_loader.js frontend/javascripts/oxalis/api/api_latest.js --github --project-name \"webKnossos Frontend API\" --format html --output public/docs/frontend-api",
     "refresh-schema": "./tools/postgres/refresh_schema.sh && rm -f target/scala-2.12/src_managed/schema/com/scalableminds/webknossos/schema/Tables.scala",
+    "enable-jobs": "sed -i -e 's/jobsEnabled = false/jobsEnabled = true/g' ./conf/application.conf",
+    "disable-jobs": "sed -i -e 's/jobsEnabled = true/jobsEnabled = false/g' ./conf/application.conf",
     "coverage": "nyc report --reporter=text-lcov | coveralls",
     "postcheckout": "echo 'Deleting auto-generated flow files...' && rm -f frontend/javascripts/test/snapshots/flow-check/*.js",
     "apply-evolutions": "tools/postgres/apply_evolutions.sh",

tools/postgres/schema.sql

@@ -21,7 +21,7 @@ START TRANSACTION;
 CREATE TABLE webknossos.releaseInformation (
   schemaVersion BIGINT NOT NULL
 );
-INSERT INTO webknossos.releaseInformation(schemaVersion) values(71);
+INSERT INTO webknossos.releaseInformation(schemaVersion) values(72);
 COMMIT TRANSACTION;
 
 
@@ -360,13 +360,16 @@ CREATE TABLE webknossos.maintenance(
 );
 INSERT INTO webknossos.maintenance(maintenanceExpirationTime) values('2000-01-01 00:00:00');
 
+CREATE TYPE webknossos.JOB_MANUAL_STATE AS ENUM ('SUCCESS', 'FAILURE');
+
 CREATE TABLE webknossos.jobs(
   _id CHAR(24) PRIMARY KEY DEFAULT '',
   _owner CHAR(24) NOT NULL,
   command TEXT NOT NULL,
   commandArgs JSONB NOT NULL,
   celeryJobId CHAR(36) NOT NULL,
   celeryInfo JSONB NOT NULL,
+  manualState webknossos.JOB_MANUAL_STATE,
   created TIMESTAMPTZ NOT NULL DEFAULT NOW(),
   isDeleted BOOLEAN NOT NULL DEFAULT false
 );