Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix importing a dataset from disk #6615

Merged
merged 6 commits into from
Nov 9, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.unreleased.md
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released
- Re-phrased some backend (error) messages to improve clarity and provide helping hints. [#6616](https://github.com/scalableminds/webknossos/pull/6616)

### Fixed
- Fixed importing a dataset from disk. [#6615](https://github.com/scalableminds/webknossos/pull/6615)

### Removed

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -169,8 +169,8 @@ function DatasetAddZarrView(props: Props) {
{/* Only the component's visibility is changed, so that the form is always rendered.
This is necessary so that the form's structure is always populated. */}
<DatasetSettingsDataTab
isEditingMode={false}
isReadOnlyDataset={false}
allowRenamingDataset
form={form}
activeDataSourceEditMode={dataSourceEditMode}
onChange={(activeEditMode) => {
Expand Down
5 changes: 3 additions & 2 deletions frontend/javascripts/admin/dataset/dataset_components.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ export const layerNameRules = [

export const getDatasetNameRules = (
activeUser: APIUser | null | undefined,
isEditing: boolean = false,
allowRenaming: boolean = true,
) => [
{
required: true,
Expand All @@ -61,7 +61,8 @@ export const getDatasetNameRules = (
...layerNameRules,
{
validator: async (_rule: any, value: string) => {
if (isEditing) {
if (!allowRenaming) {
// Renaming is not allowed. No need to validate the (existing) name then.
return Promise.resolve();
}
if (!activeUser) throw new Error("Can't do operation if no user is logged in.");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ import { useSelector } from "react-redux";
import { DeleteOutlined } from "@ant-design/icons";
import { APIDataLayer, APIDatasetId } from "types/api_flow_types";
import { useStartAndPollJob } from "admin/job/job_hooks";
import { Vector3 } from "oxalis/constants";

const FormItem = Form.Item;

Expand Down Expand Up @@ -62,15 +63,15 @@ export const syncDataSourceFields = (
};

export default function DatasetSettingsDataTab({
isEditingMode,
allowRenamingDataset,
isReadOnlyDataset,
form,
activeDataSourceEditMode,
onChange,
additionalAlert,
datasetId,
}: {
isEditingMode: boolean;
allowRenamingDataset: boolean;
isReadOnlyDataset: boolean;
form: FormInstance;
activeDataSourceEditMode: "simple" | "advanced";
Expand Down Expand Up @@ -131,7 +132,7 @@ export default function DatasetSettingsDataTab({
<RetryingErrorBoundary>
<SimpleDatasetForm
datasetId={datasetId}
isEditingMode={isEditingMode}
allowRenamingDataset={allowRenamingDataset}
isReadOnlyDataset={isReadOnlyDataset}
form={form}
dataSource={dataSource}
Expand Down Expand Up @@ -162,13 +163,13 @@ export default function DatasetSettingsDataTab({
}

function SimpleDatasetForm({
isEditingMode,
allowRenamingDataset,
isReadOnlyDataset,
dataSource,
form,
datasetId,
}: {
isEditingMode: boolean;
allowRenamingDataset: boolean;
isReadOnlyDataset: boolean;
dataSource: Record<string, any>;
form: FormInstance;
Expand Down Expand Up @@ -213,11 +214,11 @@ function SimpleDatasetForm({
label="Name"
info="The name of the dataset"
validateFirst
rules={getDatasetNameRules(activeUser, isEditingMode)}
rules={getDatasetNameRules(activeUser, allowRenamingDataset)}
>
<Input
// Renaming an existing DS is not supported right now
disabled={isReadOnlyDataset || isEditingMode}
disabled={isReadOnlyDataset || !allowRenamingDataset}
style={{
width: 400,
}}
Expand All @@ -236,8 +237,7 @@ function SimpleDatasetForm({
},
{
validator: syncValidator(
// @ts-expect-error ts-migrate(7006) FIXME: Parameter 'el' implicitly has an 'any' type.
(value) => value && value.every((el) => el > 0),
(value: Vector3) => value && value.every((el) => el > 0),
"Each component of the scale must be greater than 0",
),
},
Expand Down Expand Up @@ -504,16 +504,17 @@ function SimpleLayerForm({
</Select>
</Form.Item>

{/* The in-condition is only necessary to satisfy TypeScript */}
{isSegmentation && "largestSegmentId" in layer ? (
{isSegmentation ? (
<div>
<div style={{ display: "flex", alignItems: "end" }}>
<FormItemWithInfo
name={["dataSource", "dataLayers", index, "largestSegmentId"]}
label="Largest segment ID"
info="The largest segment ID specifies the highest id which exists in this segmentation layer. When users extend this segmentation, new IDs will be assigned starting from that value."
initialValue={
layer.largestSegmentId != null ? `${layer.largestSegmentId}` : undefined
"largestSegmentId" in layer && layer.largestSegmentId != null
? `${layer.largestSegmentId}`
: undefined
}
rules={[
{
Expand Down
43 changes: 29 additions & 14 deletions frontend/javascripts/dashboard/dataset/dataset_settings_view.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,9 @@ import type {
MutableAPIDataSource,
APIDatasetId,
APIMessage,
APIUnimportedDatasource,
} from "types/api_flow_types";
import { Unicode } from "oxalis/constants";
import { Unicode, Vector3 } from "oxalis/constants";
import type { DatasetConfiguration, OxalisState } from "oxalis/store";
import DatasetCacheProvider, { datasetCache } from "dashboard/dataset/dataset_cache_provider";
import LinkButton from "components/link_button";
Expand Down Expand Up @@ -95,7 +96,7 @@ type State = {
isLoading: boolean;
activeDataSourceEditMode: "simple" | "advanced";
activeTabKey: TabKey;
savedDataSourceOnServer: APIDataSource | null | undefined;
savedDataSourceOnServer: APIDataSource | APIUnimportedDatasource | null | undefined;
inferredDataSource: APIDataSource | null | undefined;
differenceBetweenDataSources: Record<string, any>;
hasNoAllowedTeams: boolean;
Expand All @@ -110,25 +111,35 @@ export type FormData = {
};

function ensureValidScaleOnInferredDataSource(
savedDataSourceOnServer: APIDataSource | null | undefined,
savedDataSourceOnServer: APIDataSource | APIUnimportedDatasource | null | undefined,
inferredDataSource: APIDataSource | null | undefined,
): APIDataSource | null | undefined {
if (savedDataSourceOnServer == null || inferredDataSource == null) {
// If one of the data sources is null, return the other.
return savedDataSourceOnServer || inferredDataSource;
const potentialSource = savedDataSourceOnServer || inferredDataSource;
if (potentialSource && "dataLayers" in potentialSource) {
return potentialSource;
} else {
return null;
}
}

const inferredDataSourceClone = _.cloneDeep(inferredDataSource) as any as MutableAPIDataSource;

if (
_.isEqual(inferredDataSource.scale, [0, 0, 0]) &&
!_.isEqual(savedDataSourceOnServer.scale, [0, 0, 0])
) {
inferredDataSourceClone.scale = savedDataSourceOnServer.scale;
const savedScale =
"dataLayers" in savedDataSourceOnServer
? savedDataSourceOnServer.scale
: ([0, 0, 0] as Vector3);
if (_.isEqual(inferredDataSource.scale, [0, 0, 0]) && !_.isEqual(savedScale, [0, 0, 0])) {
inferredDataSourceClone.scale = savedScale;
}

// Trying to use the saved value for largestSegmentId instead of 0.
if (savedDataSourceOnServer.dataLayers != null && inferredDataSourceClone.dataLayers != null) {
if (
"dataLayers" in savedDataSourceOnServer &&
savedDataSourceOnServer.dataLayers != null &&
inferredDataSourceClone.dataLayers != null
) {
const segmentationLayerSettings = inferredDataSourceClone.dataLayers.find(
(layer) => layer.category === "segmentation",
);
Expand Down Expand Up @@ -250,7 +261,11 @@ class DatasetSettingsView extends React.PureComponent<PropsWithFormAndRouter, St

// Ensure that zarr layers (which aren't inferred by the back-end) are still
// included in the inferred data source
if (savedDataSourceOnServer != null && inferredDataSource != null) {
if (
savedDataSourceOnServer &&
"dataLayers" in savedDataSourceOnServer &&
inferredDataSource != null
) {
const layerDict = _.keyBy(inferredDataSource.dataLayers, "name");
for (const layer of savedDataSourceOnServer.dataLayers) {
if (layer.dataFormat === "zarr" && layerDict[layer.name] == null) {
Expand Down Expand Up @@ -425,8 +440,8 @@ class DatasetSettingsView extends React.PureComponent<PropsWithFormAndRouter, St
message = (
<div>
The current datasource-properties.json on the server seems to be in an invalid JSON format
and webKnossos could not parse this file. The settings below are suggested by webKnossos
and should be adjusted. <br />
(or is missing completely). The settings below are suggested by webKnossos and should be
adjusted. <br />
Be aware that webKnossos cannot guess properties like the voxel size or the largest
segment id. You must set them yourself.
</div>
Expand Down Expand Up @@ -854,7 +869,7 @@ class DatasetSettingsView extends React.PureComponent<PropsWithFormAndRouter, St
<DatasetSettingsDataTab
key="SimpleAdvancedDataForm"
datasetId={this.props.datasetId}
isEditingMode={this.props.isEditingMode}
allowRenamingDataset={false}
isReadOnlyDataset={
this.state.dataset != null && this.state.dataset.dataStore.isConnector
}
Expand Down
4 changes: 2 additions & 2 deletions frontend/javascripts/types/api_flow_types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ type MutableAPIDataSourceBase = {
status?: string;
};
type APIDataSourceBase = Readonly<MutableAPIDataSourceBase>;
type APIUnimportedDatasource = APIDataSourceBase;
export type APIUnimportedDatasource = APIDataSourceBase;
export type MutableAPIDataSource = MutableAPIDataSourceBase & {
dataLayers: Array<APIDataLayer>;
scale: Vector3;
Expand Down Expand Up @@ -148,7 +148,7 @@ type APIUnimportedDataset = APIDatasetBase & {
export type APIMaybeUnimportedDataset = APIUnimportedDataset | APIDataset;
export type APIDataSourceWithMessages = {
readonly dataSource?: APIDataSource;
readonly previousDataSource?: APIDataSource;
readonly previousDataSource?: APIDataSource | APIUnimportedDatasource;
readonly messages: Array<APIMessage>;
};
export type APITeamMembership = {
Expand Down