Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add neuron reconstruction job backend and frontend part #5922

Merged
merged 35 commits into from
Feb 10, 2022
Merged
Show file tree
Hide file tree
Changes from 18 commits
Commits
Show all changes
35 commits
Select commit Hold shift + click to select a range
b0d50ae
add backend part of new nuclei reconstruction job
MichaelBuessemeyer Dec 22, 2021
920d07c
add frontend part of new nuclei reconstruction job
MichaelBuessemeyer Dec 22, 2021
72a5e4c
fix capitilazation of job name
MichaelBuessemeyer Dec 22, 2021
c93a4d6
Merge branch 'master' of github.com:scalableminds/webknossos into add…
MichaelBuessemeyer Jan 3, 2022
290633a
rename nuclei reconstruction to neuron reconstruction
MichaelBuessemeyer Jan 3, 2022
bc7d3ce
fix neuron reconstruction job request path in frontend
MichaelBuessemeyer Jan 3, 2022
622d54b
rename to neuron inferral
MichaelBuessemeyer Jan 6, 2022
fb05816
Merge branch 'master' of github.com:scalableminds/webknossos into add…
MichaelBuessemeyer Jan 6, 2022
ef0f324
format backend
MichaelBuessemeyer Jan 10, 2022
17f2fc9
WIP: Make bounding box of neuron inferral configurable
MichaelBuessemeyer Jan 10, 2022
2d62222
Add bounding box selection for neuron inferral job
MichaelBuessemeyer Jan 13, 2022
3e0f253
Merge branch 'master' of github.com:scalableminds/webknossos into add…
MichaelBuessemeyer Jan 13, 2022
42d8478
remove unused import and outdated comment
MichaelBuessemeyer Jan 13, 2022
5d63776
fix css class name
MichaelBuessemeyer Jan 13, 2022
8bbe7ff
Only show neuron inferral job option to admins
MichaelBuessemeyer Jan 20, 2022
904d52b
small fixes before review
MichaelBuessemeyer Jan 24, 2022
d5a578f
Merge branch 'master' into add-nuclei-reconstruction-job
MichaelBuessemeyer Jan 24, 2022
bbff804
disable jobs
MichaelBuessemeyer Jan 24, 2022
cd9a2ae
Add result link
fm3 Jan 25, 2022
247146a
Merge origin/master into this branch
MichaelBuessemeyer Jan 27, 2022
8abc528
add example image for neuron inferral
MichaelBuessemeyer Jan 27, 2022
14387c8
Merge branch 'add-nuclei-reconstruction-job' of github.com:scalablemi…
MichaelBuessemeyer Jan 27, 2022
7720c3a
apply review feedback
MichaelBuessemeyer Jan 27, 2022
3558115
format backend && add eslintcache to clean script
MichaelBuessemeyer Jan 27, 2022
534a585
Merge branch 'master' of github.com:scalableminds/webknossos into add…
MichaelBuessemeyer Jan 27, 2022
6eee2ae
apply pr feedback
MichaelBuessemeyer Jan 31, 2022
f21c2f9
Merge branch 'master' of github.com:scalableminds/webknossos into add…
MichaelBuessemeyer Jan 31, 2022
e3616b7
Apply suggestions from code review
MichaelBuessemeyer Feb 3, 2022
32a3f8d
Merge branch 'master' into add-nuclei-reconstruction-job
MichaelBuessemeyer Feb 3, 2022
2d76a44
fix formatting
MichaelBuessemeyer Feb 3, 2022
369ff5f
Merge branch 'add-nuclei-reconstruction-job' of github.com:scalablemi…
MichaelBuessemeyer Feb 3, 2022
46629d6
Include isSuperUser in user json
fm3 Feb 3, 2022
753b0e3
make neuron inferral only available to super users
MichaelBuessemeyer Feb 7, 2022
9019ecc
Merge branch 'master' of github.com:scalableminds/webknossos into add…
MichaelBuessemeyer Feb 10, 2022
5267721
Merge branch 'master' into add-nuclei-reconstruction-job
MichaelBuessemeyer Feb 10, 2022
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 27 additions & 0 deletions app/controllers/JobsController.scala
Original file line number Diff line number Diff line change
Expand Up @@ -149,6 +149,33 @@ class JobsController @Inject()(jobDAO: JobDAO,
}
}

def runInferNeuronsJob(organizationName: String,
dataSetName: String,
layerName: String,
bbox: String): Action[AnyContent] =
sil.SecuredAction.async { implicit request =>
log(Some(slackNotificationService.noticeFailedJobRequest)) {
for {
organization <- organizationDAO.findOneByName(organizationName) ?~> Messages("organization.notFound",
organizationName)
_ <- bool2Fox(request.identity._organization == organization._id) ?~> "job.inferNeurons.notAllowed.organization" ~> FORBIDDEN
dataSet <- dataSetDAO.findOneByNameAndOrganization(dataSetName, organization._id) ?~> Messages(
"dataSet.notFound",
dataSetName) ~> NOT_FOUND
command = "infer_neurons"
commandArgs = Json.obj(
"organization_name" -> organizationName,
"dataset_name" -> dataSetName,
"layer_name" -> layerName,
"webknossos_token" -> RpcTokenHolder.webKnossosToken,
"bbox" -> bbox,
)
job <- jobService.submitJob(command, commandArgs, request.identity, dataSet._dataStore) ?~> "job.couldNotRunNeuronInferral"
js <- jobService.publicWrites(job)
} yield Ok(js)
}
}

def runExportTiffJob(organizationName: String,
dataSetName: String,
bbox: String,
Expand Down
2 changes: 2 additions & 0 deletions conf/messages
Original file line number Diff line number Diff line change
Expand Up @@ -337,6 +337,7 @@ job.couldNotRunCubing = Failed to start WKW conversion job.
job.couldNotRunTiffExport = Failed to start Tiff export job.
job.couldNotRunComputeMeshFile = Failed to start mesh file computation job.
job.couldNotRunNucleiInferral = Failed to start nuclei inferral job.
job.couldNotRunNeuronInferral = Failed to start neuron inferral job.
job.disabled = Long-running jobs are not enabled for this webKnossos instance.
jobs.worker.notFound = Could not find this worker in the database.
job.export.fileNotFound = Exported file not found. The link may be expired.
Expand All @@ -345,6 +346,7 @@ job.export.tiff.volumeExceeded = The volume of the selected bounding box is too
job.export.tiff.edgeLengthExceeded = An edge length of the selected bounding box is too large.
job.export.notAllowed.organization = Currently tiff export is only allowed for datasets of your own organization.
job.inferNuclei.notAllowed.organization = Currently nuclei inferral is only allowed for datasets of your own organization.
job.inferNeurons.notAllowed.organization = Currently neuron inferral is only allowed for datasets of your own organization.
job.meshFile.notAllowed.organization = Calculating mesh files is only allowed for datasets of your own organization.

agglomerateSkeleton.failed=Could not generate agglomerate skeleton.
Expand Down
1 change: 1 addition & 0 deletions conf/webknossos.latest.routes
Original file line number Diff line number Diff line change
Expand Up @@ -206,6 +206,7 @@ GET /jobs/run/convertToWkw/:organizationName/:dataSetName c
GET /jobs/run/computeMeshFile/:organizationName/:dataSetName controllers.JobsController.runComputeMeshFileJob(organizationName: String, dataSetName: String, layerName: String, mag: String, agglomerateView: Option[String])
GET /jobs/run/exportTiff/:organizationName/:dataSetName controllers.JobsController.runExportTiffJob(organizationName: String, dataSetName: String, bbox: String, layerName: Option[String], tracingId: Option[String], tracingVersion: Option[String], annotationId: Option[String], annotationType: Option[String], hideUnmappedIds: Option[Boolean], mappingName: Option[String], mappingType: Option[String])
GET /jobs/run/inferNuclei/:organizationName/:dataSetName controllers.JobsController.runInferNucleiJob(organizationName: String, dataSetName: String, layerName: Option[String])
GET /jobs/run/inferNeurons/:organizationName/:dataSetName controllers.JobsController.runInferNeuronsJob(organizationName: String, dataSetName: String, layerName: String, bbox: String)
GET /jobs/:id controllers.JobsController.get(id: String)
PATCH /jobs/:id/cancel controllers.JobsController.cancel(id: String)
POST /jobs/:id/status controllers.WKRemoteWorkerController.updateJobStatus(key: String, id: String)
13 changes: 13 additions & 0 deletions frontend/javascripts/admin/admin_rest_api.js
Original file line number Diff line number Diff line change
Expand Up @@ -1039,6 +1039,19 @@ export function startNucleiInferralJob(
);
}

export function startNeuronInferralJob(
organizationName: string,
datasetName: string,
layerName: string,
bbox: Vector6,
): Promise<APIJob> {
return Request.receiveJSON(
`/api/jobs/run/inferNeurons/${organizationName}/${datasetName}?layerName=${layerName}&bbox=${bbox.join(
",",
)}`,
);
}

export function getDatasetDatasource(
dataset: APIMaybeUnimportedDataset,
): Promise<APIDataSourceWithMessages> {
Expand Down
14 changes: 14 additions & 0 deletions frontend/javascripts/admin/job/job_list_view.js
Original file line number Diff line number Diff line change
Expand Up @@ -230,6 +230,20 @@ class JobListView extends React.PureComponent<Props, State> {
)}
</span>
);
} else if (job.type === "reconstruct_neurons") {
return (
<span>
{job.state === "SUCCESS" && job.result && job.organizationName && (
<Link
to={`/datasets/${job.organizationName}/${job.result}/view`}
title="View Segmentation"
>
<EyeOutlined />
View
</Link>
)}
</span>
);
MichaelBuessemeyer marked this conversation as resolved.
Show resolved Hide resolved
} else return null;
};

Expand Down
7 changes: 7 additions & 0 deletions frontend/javascripts/libs/utils.js
Original file line number Diff line number Diff line change
Expand Up @@ -163,6 +163,13 @@ export function capitalize(str: string): string {
return str[0].toUpperCase() + str.slice(1);
}

export function capitalizeWords(str: string): string {
return str
.split(" ")
.map(capitalize)
.join(" ");
}

function intToHex(int: number, digits: number = 6): string {
return (_.repeat("0", digits) + int.toString(16)).slice(-digits);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import type {
ReadOnlyTracing,
SkeletonTracing,
Tracing,
UserBoundingBox,
VolumeTracing,
} from "oxalis/store";
import { type ServerTracing, type TracingType, TracingTypeEnum } from "types/api_flow_types";
Expand Down Expand Up @@ -69,3 +70,8 @@ export function selectTracing(

return volumeTracing;
}

export const getUserBoundingBoxesFromState = (state: OxalisState): Array<UserBoundingBox> => {
const maybeSomeTracing = maybeGetSomeTracing(state.tracing);
return maybeSomeTracing != null ? maybeSomeTracing.userBoundingBoxes : [];
};
10 changes: 4 additions & 6 deletions frontend/javascripts/oxalis/model/sagas/save_saga.js
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,10 @@ import {
} from "oxalis/model/accessors/volumetracing_accessor";
import { getResolutionInfo } from "oxalis/model/accessors/dataset_accessor";
import { globalPositionToBucketPosition } from "oxalis/model/helpers/position_converter";
import { maybeGetSomeTracing, selectTracing } from "oxalis/model/accessors/tracing_accessor";
import {
getUserBoundingBoxesFromState,
selectTracing,
} from "oxalis/model/accessors/tracing_accessor";
import { selectQueue } from "oxalis/model/accessors/save_accessor";
import { setBusyBlockingInfoAction } from "oxalis/model/actions/ui_actions";
import Date from "libs/date";
Expand Down Expand Up @@ -171,11 +174,6 @@ function unpackRelevantActionForUndo(action): RelevantActionsForUndoRedo {
throw new Error("Could not unpack redux action from channel");
}

const getUserBoundingBoxesFromState = state => {
const maybeSomeTracing = maybeGetSomeTracing(state.tracing);
return maybeSomeTracing != null ? maybeSomeTracing.userBoundingBoxes : [];
};

export function* collectUndoStates(): Saga<void> {
const undoStack: Array<UndoState> = [];
const redoStack: Array<UndoState> = [];
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -507,15 +507,9 @@ export class ColorSetting extends React.PureComponent<ColorSettingPropTypes> {
style = style || {};
return (
<div
id="color-picker-wrapper"
philippotto marked this conversation as resolved.
Show resolved Hide resolved
className="color-display-wrapper"
style={{
backgroundColor: value,
display: "inline-block",
width: 16,
height: 16,
borderRadius: 3,
boxShadow: "0px 0px 3px #cacaca",
verticalAlign: "middle",
...style,
}}
>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,15 @@ import EditableTextLabel from "oxalis/view/components/editable_text_label";
import Model from "oxalis/model";
import features from "features";
import Store, { type OxalisState, type Task, type Tracing } from "oxalis/store";
import NucleiInferralModal from "oxalis/view/right-border-tabs/nuclei_inferral_modal";
import {
NucleiInferralModal,
NeuronInferralModal,
} from "oxalis/view/right-border-tabs/starting_job_modals";

const StartableJobsEnum = {
NUCLEI_INFERRAL: "nuclei inferral",
NEURON_INFERRAL: "neuron inferral",
};

type StateProps = {|
tracing: Tracing,
Expand All @@ -49,7 +57,7 @@ type DispatchProps = {|
type Props = {| ...StateProps, ...DispatchProps |};

type State = {
showNucleiInferralModal: boolean,
showJobsDetailsModal: ?$Values<typeof StartableJobsEnum>,
};

const shortcuts = [
Expand Down Expand Up @@ -134,7 +142,7 @@ export function convertPixelsToNm(

class DatasetInfoTabView extends React.PureComponent<Props, State> {
state = {
showNucleiInferralModal: false,
showJobsDetailsModal: null,
};

setAnnotationName = (newName: string) => {
Expand Down Expand Up @@ -200,7 +208,8 @@ class DatasetInfoTabView extends React.PureComponent<Props, State> {
}

getProcessingJobsMenu = () => {
if (!this.props.dataset.jobsEnabled) {
const { dataset } = this.props;
if (!dataset.jobsEnabled) {
return (
<tr>
<td style={{ paddingRight: 4, paddingTop: 10, verticalAlign: "top" }}>
Expand All @@ -216,21 +225,34 @@ class DatasetInfoTabView extends React.PureComponent<Props, State> {
</tr>
);
}
const overlay = (
<Menu>
<Menu.Item onClick={() => this.setState({ showNucleiInferralModal: true })}>
<Tooltip title="Start a job that automatically detects nuclei for this dataset.">
Start Nuclei Inferral
const jobMenuItems = [
<Menu.Item
key="start_nuclei_inferal"
onClick={() => this.setState({ showJobsDetailsModal: StartableJobsEnum.NUCLEI_INFERRAL })}
>
<Tooltip title="Start a job that automatically detects nuclei for this dataset.">
Start Nuclei Inferral
</Tooltip>
</Menu.Item>,
];
if (this.props.activeUser != null && this.props.activeUser.isAdmin) {
jobMenuItems.push(
<Menu.Item
key="start_neuron_inferral"
onClick={() => this.setState({ showJobsDetailsModal: StartableJobsEnum.NEURON_INFERRAL })}
>
<Tooltip title="Start a job that automatically reconstructs neurons for this dataset.">
Start Neuron Inferral
</Tooltip>
</Menu.Item>
</Menu>
);
</Menu.Item>,
);
}
return (
<tr>
<td style={{ paddingRight: 4, paddingTop: 10, verticalAlign: "top" }}>
<StarOutlined className="info-tab-icon" style={{ fontSize: 18 }} />
</td>
<Dropdown overlay={overlay} overlayStyle={{ minWidth: "unset" }}>
<Dropdown overlay={<Menu>{jobMenuItems}</Menu>} overlayStyle={{ minWidth: "unset" }}>
<td>
<Button type="link" style={{ padding: 0 }}>
Process Dataset
Expand Down Expand Up @@ -434,6 +456,16 @@ class DatasetInfoTabView extends React.PureComponent<Props, State> {
return null;
}

renderSelectedStartingJobsModal() {
const handleClose = () => this.setState({ showJobsDetailsModal: null });
if (this.state.showJobsDetailsModal === StartableJobsEnum.NUCLEI_INFERRAL) {
return <NucleiInferralModal handleClose={handleClose} />;
} else if (this.state.showJobsDetailsModal === StartableJobsEnum.NEURON_INFERRAL) {
return <NeuronInferralModal handleClose={handleClose} />;
}
return null;
}

render() {
const { dataset, activeResolution, activeUser } = this.props;
const isDatasetViewMode =
Expand Down Expand Up @@ -523,12 +555,7 @@ class DatasetInfoTabView extends React.PureComponent<Props, State> {
: null}
</tbody>
</table>
{this.state.showNucleiInferralModal ? (
<NucleiInferralModal
dataset={dataset}
handleClose={() => this.setState({ showNucleiInferralModal: false })}
/>
) : null}
{this.renderSelectedStartingJobsModal()}
</div>

<div className="info-tab-block">{this.getTracingStatistics()}</div>
Expand Down
Loading