Skip to content

Commit

Permalink
Merge pull request #5091 from specify/issue-4980
Browse files Browse the repository at this point in the history
Add a feature to choose type of taxon tree in WB
acwhite211 authored Sep 11, 2024

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature.
2 parents f58f0d4 + eb3c820 commit dfa30a6
Showing 22 changed files with 875 additions and 174 deletions.
Original file line number Diff line number Diff line change
@@ -24,5 +24,6 @@ test('domain data is fetched and parsed correctly', async () =>
],
paleoContextChildTable: 'collectionobject',
referenceSymbol: '#',
treeSymbol: '$',
treeRankSymbol: '$',
treeDefinitionSymbol: '%',
}));
7 changes: 5 additions & 2 deletions specifyweb/frontend/js_src/lib/components/DataModel/schema.ts
Original file line number Diff line number Diff line change
@@ -29,7 +29,8 @@ type Schema = {
'Institution'
];
readonly referenceSymbol: string;
readonly treeSymbol: string;
readonly treeDefinitionSymbol: string;
readonly treeRankSymbol: string;
readonly fieldPartSeparator: string;
};

@@ -60,8 +61,10 @@ const schemaBase: Writable<Schema> = {

// Prefix for -to-many indexes
referenceSymbol: '#',
// Prefix for Tree Definitions
treeDefinitionSymbol: '%',
// Prefix for tree ranks
treeSymbol: '$',
treeRankSymbol: '$',
// Separator for partial fields (date parts in Query Builder)
fieldPartSeparator: '-',
};
Original file line number Diff line number Diff line change
@@ -118,11 +118,14 @@ export function getTreeDefinitions<TREE_NAME extends AnyTree['tableName']>(
tableName
);

return typeof treeDefinitionId === 'number'
? specificTreeDefinitions.filter(
({ definition }) => definition.id === treeDefinitionId
)
: specificTreeDefinitions;
if (typeof treeDefinitionId === 'number') {
const resolvedDefinition = specificTreeDefinitions.find(
({ definition }) => definition.id === treeDefinitionId
);
return resolvedDefinition === undefined
? specificTreeDefinitions
: [resolvedDefinition];
} else return specificTreeDefinitions;
}

export function getTreeDefinitionItems<TREE_NAME extends AnyTree['tableName']>(
Original file line number Diff line number Diff line change
@@ -94,8 +94,8 @@ test('Editing Data Object Formatter', () => {
</switch>
</format>
<aggregators>
<aggregator name='AccessionAgent' title='AccessionAgent' class='edu.ku.brc.specify.datamodel.Agent' default='true' separator='' ending='' format='AccessionAgent'/>
<aggregator name='AccessionAgent' title='AccessionAgent' class='edu.ku.brc.specify.datamodel.AccessionAgent' default='true' separator='' ending='' format='AccessionAgent'/>
<aggregator name='AccessionAgent' title='AccessionAgent' class='edu.ku.brc.specify.datamodel.Agent' default='true' separator='; ' ending='' format='AccessionAgent'/>
<aggregator name='AccessionAgent' title='AccessionAgent' class='edu.ku.brc.specify.datamodel.AccessionAgent' default='true' separator='; ' ending='' format='AccessionAgent'/>
</aggregators>
</formatters>"
`);
Original file line number Diff line number Diff line change
@@ -37,7 +37,6 @@ export type HtmlGeneratorFieldData = {
readonly isDefault?: boolean;
readonly isRelationship?: boolean;
readonly tableName?: keyof Tables;
readonly tableTreeDefName?: string;
};

type MappingLineBaseProps = {
Original file line number Diff line number Diff line change
@@ -16,16 +16,16 @@ theories(valueIsToManyIndex, [
[[`${schema.referenceSymbol}2`], true],
[[`${schema.referenceSymbol}999`], true],
[['collectionobject'], false],
[[`${schema.treeSymbol}Kingdom`], false],
[[`${schema.treeRankSymbol}Kingdom`], false],
]);

theories(valueIsTreeRank, [
[[`${schema.referenceSymbol}1`], false],
[[`${schema.referenceSymbol}2`], false],
[[`${schema.referenceSymbol}999`], false],
[['collectionobject'], false],
[[`${schema.treeSymbol}Kingdom`], true],
[[`${schema.treeSymbol}County`], true],
[[`${schema.treeRankSymbol}Kingdom`], true],
[[`${schema.treeRankSymbol}County`], true],
]);

theories(getNumberFromToManyIndex, [
@@ -36,8 +36,8 @@ theories(getNumberFromToManyIndex, [
]);

theories(getNameFromTreeRankName, [
[[`${schema.treeSymbol}Kingdom`], 'Kingdom'],
[[`${schema.treeSymbol}County`], 'County'],
[[`${schema.treeRankSymbol}Kingdom`], 'Kingdom'],
[[`${schema.treeRankSymbol}County`], 'County'],
]);

theories(findDuplicateMappings, [
Original file line number Diff line number Diff line change
@@ -314,70 +314,60 @@ theories(getMappingLineData, [
isRelationship: true,
isDefault: false,
tableName: 'Taxon',
tableTreeDefName: 'Taxonomy',
},
$Phylum: {
optionLabel: 'Phylum',
isRelationship: true,
isDefault: false,
tableName: 'Taxon',
tableTreeDefName: 'Taxonomy',
},
$Class: {
optionLabel: 'Class',
isRelationship: true,
isDefault: false,
tableName: 'Taxon',
tableTreeDefName: 'Taxonomy',
},
$Order: {
optionLabel: 'Order',
isRelationship: true,
isDefault: false,
tableName: 'Taxon',
tableTreeDefName: 'Taxonomy',
},
$Family: {
optionLabel: 'Family',
isRelationship: true,
isDefault: true,
tableName: 'Taxon',
tableTreeDefName: 'Taxonomy',
},
$Subfamily: {
optionLabel: 'Subfamily',
isRelationship: true,
isDefault: false,
tableName: 'Taxon',
tableTreeDefName: 'Taxonomy',
},
$Genus: {
optionLabel: 'Genus',
isRelationship: true,
isDefault: false,
tableName: 'Taxon',
tableTreeDefName: 'Taxonomy',
},
$Subgenus: {
optionLabel: 'Subgenus',
isRelationship: true,
isDefault: false,
tableName: 'Taxon',
tableTreeDefName: 'Taxonomy',
},
$Species: {
optionLabel: 'Species',
isRelationship: true,
isDefault: false,
tableName: 'Taxon',
tableTreeDefName: 'Taxonomy',
},
$Subspecies: {
optionLabel: 'Subspecies',
isRelationship: true,
isDefault: false,
tableName: 'Taxon',
tableTreeDefName: 'Taxonomy',
},
},
tableName: 'Taxon',
@@ -870,70 +860,60 @@ theories(getMappingLineData, [
isRelationship: true,
optionLabel: 'Class',
tableName: 'Taxon',
tableTreeDefName: 'Taxonomy',
},
$Family: {
isDefault: true,
isRelationship: true,
optionLabel: 'Family',
tableName: 'Taxon',
tableTreeDefName: 'Taxonomy',
},
$Genus: {
isDefault: false,
isRelationship: true,
optionLabel: 'Genus',
tableName: 'Taxon',
tableTreeDefName: 'Taxonomy',
},
$Kingdom: {
isDefault: false,
isRelationship: true,
optionLabel: 'Kingdom',
tableName: 'Taxon',
tableTreeDefName: 'Taxonomy',
},
$Order: {
isDefault: false,
isRelationship: true,
optionLabel: 'Order',
tableName: 'Taxon',
tableTreeDefName: 'Taxonomy',
},
$Phylum: {
isDefault: false,
isRelationship: true,
optionLabel: 'Phylum',
tableName: 'Taxon',
tableTreeDefName: 'Taxonomy',
},
$Species: {
isDefault: false,
isRelationship: true,
optionLabel: 'Species',
tableName: 'Taxon',
tableTreeDefName: 'Taxonomy',
},
$Subfamily: {
isDefault: false,
isRelationship: true,
optionLabel: 'Subfamily',
tableName: 'Taxon',
tableTreeDefName: 'Taxonomy',
},
$Subgenus: {
isDefault: false,
isRelationship: true,
optionLabel: 'Subgenus',
tableName: 'Taxon',
tableTreeDefName: 'Taxonomy',
},
$Subspecies: {
isDefault: false,
isRelationship: true,
optionLabel: 'Subspecies',
tableName: 'Taxon',
tableTreeDefName: 'Taxonomy',
},
},
selectLabel: localized('Taxon'),
Original file line number Diff line number Diff line change
@@ -29,9 +29,20 @@ export const valueIsToManyIndex = (value: string | undefined): boolean =>
value?.slice(0, schema.referenceSymbol.length) === schema.referenceSymbol ||
false;

/**
* Returns whether a value is any special tree-related meta information
* (e.g. tree defintiion, tree rank)
*/
export const valueIsTreeMeta = (value: string | undefined): boolean =>
valueIsTreeDefinition(value) || valueIsTreeRank(value);

/** Returns whether a value is a tree definition name (e.x %Taxonomy) */
export const valueIsTreeDefinition = (value: string | undefined): boolean =>
value?.startsWith(schema.treeDefinitionSymbol) ?? false;

/** Returns whether a value is a tree rank name (e.x $Kingdom, $Order) */
export const valueIsTreeRank = (value: string | undefined): boolean =>
value?.startsWith(schema.treeSymbol) || false;
value?.startsWith(schema.treeRankSymbol) ?? false;

/**
* Returns index from a formatted -to-many index value (e.x #1 => 1)
@@ -40,6 +51,14 @@ export const valueIsTreeRank = (value: string | undefined): boolean =>
export const getNumberFromToManyIndex = (value: string): number =>
Number(value.slice(schema.referenceSymbol.length));

/**
* Returns tree definition name from a complete tree definition name
* (e.x %Taxonomy => Taxonomy)
* Opposite of formatTreeDefinition
*/
export const getNameFromTreeDefinitionName = (value: string): string =>
value.slice(schema.treeDefinitionSymbol.length);

/*
* BUG: in places where output of this function is displayed to the user,
* make sure to use tree rank title instead of name
@@ -51,7 +70,7 @@ export const getNumberFromToManyIndex = (value: string): number =>
*
*/
export const getNameFromTreeRankName = (value: string): string =>
value.slice(schema.treeSymbol.length);
value.slice(schema.treeRankSymbol.length);

/**
* Returns a formatted -to-many index from an index (e.x 1 => #1)
@@ -61,7 +80,7 @@ export const formatToManyIndex = (index: number): string =>
`${schema.referenceSymbol}${index}`;

// Meta fields
export const anyTreeRank = `${schema.fieldPartSeparator}any`;
export const anyTreeRank = `${schema.fieldPartSeparator}any` as const;
export const formattedEntry = `${schema.fieldPartSeparator}formatted`;

/**
@@ -75,14 +94,22 @@ export const formattedEntry = `${schema.fieldPartSeparator}formatted`;
*/
export const emptyMapping = '0';

/**
* Returns a complete tree definition name from a tree definition name
* (e.x Taxononomy => %Taxonomy)
* Opposite of getNameFromTreeDefinitionName
*/
export const formatTreeDefinition = (definitionName: string): string =>
`${schema.treeDefinitionSymbol}${definitionName}`;

/**
* Returns a complete tree rank name from a tree rank name
* (e.x Kingdom => $Kingdom)
* Opposite of getNameFromTreeRankName
*
*/
export const formatTreeRank = (rankName: string): string =>
`${schema.treeSymbol}${rankName}`;
`${schema.treeRankSymbol}${rankName}`;

// Match fields names like startDate_fullDate, but not _formatted
export const valueIsPartialField = (value: string): boolean =>
@@ -160,7 +187,7 @@ export const getGenericMappingPath = (mappingPath: MappingPath): MappingPath =>
mappingPath.filter(
(mappingPathPart) =>
!valueIsToManyIndex(mappingPathPart) &&
!valueIsTreeRank(mappingPathPart) &&
!valueIsTreeMeta(mappingPathPart) &&
mappingPathPart !== formattedEntry &&
mappingPathPart !== emptyMapping
);
Original file line number Diff line number Diff line change
@@ -22,6 +22,7 @@ import {
parsePartialField,
valueIsPartialField,
valueIsToManyIndex,
valueIsTreeDefinition,
valueIsTreeRank,
} from './mappingHelpers';
import { getMappingLineData } from './navigator';
@@ -120,16 +121,19 @@ export function generateMappingPathPreview(
: 1;
const toManyIndexFormatted = toManyIndexNumber > 1 ? toManyIndex : undefined;

const [databaseFieldName, databaseTableOrRankName, databaseParentTableName] =
mappingPathSubset([baseTableName, ...mappingPath]);
const [
databaseFieldName,
databaseTableOrRankName,
databaseParentTableOrTreeName,
] = mappingPathSubset([baseTableName, ...mappingPath]);

// Attributes parts of filedLables to each variable or creates one if empty
const [
fieldName = camelToHuman(databaseFieldName),
tableOrRankName = camelToHuman(
getNameFromTreeRankName(databaseTableOrRankName)
),
parentTableName = camelToHuman(databaseParentTableName),
parentTableOrTreeName = camelToHuman(databaseParentTableOrTreeName),
] = mappingPathSubset(fieldLabels);

const isAnyRank = databaseTableOrRankName === formatTreeRank(anyTreeRank);
@@ -163,16 +167,20 @@ export function generateMappingPathPreview(
const tableNameFormatted =
tablesToHide.has(databaseTableOrRankName) &&
databaseFieldName !== formattedEntry
? [parentTableName || tableNameNonEmpty]
? [parentTableOrTreeName || tableNameNonEmpty]
: genericTables.has(databaseTableOrRankName)
? [parentTableName, tableNameNonEmpty]
? [parentTableOrTreeName, tableNameNonEmpty]
: [tableNameNonEmpty];

return filterArray([
...(valueIsTreeRank(databaseTableOrRankName)
? [isAnyRank ? parentTableName : tableOrRankName]
? [isAnyRank ? parentTableOrTreeName : tableOrRankName]
: tableNameFormatted),
fieldNameFormatted,
...(valueIsTreeRank(databaseTableOrRankName) &&
valueIsTreeDefinition(databaseParentTableOrTreeName)
? [parentTableOrTreeName]
: []),
toManyIndexFormatted,
])
.filter(Boolean)
142 changes: 113 additions & 29 deletions specifyweb/frontend/js_src/lib/components/WbPlanView/navigator.ts
Original file line number Diff line number Diff line change
@@ -29,14 +29,17 @@ import {
formatPartialField,
formattedEntry,
formatToManyIndex,
formatTreeDefinition,
formatTreeRank,
getGenericMappingPath,
getNameFromTreeDefinitionName,
getNameFromTreeRankName,
parsePartialField,
relationshipIsToMany,
valueIsPartialField,
valueIsToManyIndex,
valueIsTreeRank,
valueIsTreeDefinition,
valueIsTreeMeta,
} from './mappingHelpers';
import { getMaxToManyIndex, isCircularRelationship } from './modelHelpers';
import type { NavigatorSpec } from './navigatorSpecs';
@@ -64,9 +67,14 @@ type NavigationCallbacks = {
readonly handleToManyChildren: (
callbackPayload: Readonly<NavigationCallbackPayload>
) => void;
readonly handleTreeDefinitions: (
callbackPayload: Readonly<NavigationCallbackPayload>
) => void;
// Handles tree ranks children
readonly handleTreeRanks: (
callbackPayload: Readonly<NavigationCallbackPayload>
callbackPayload: Readonly<NavigationCallbackPayload> & {
readonly definitionName: string;
}
) => void;
// Handles field and relationships
readonly handleSimpleFields: (
@@ -84,10 +92,12 @@ type NavigationCallbacks = {
* fields from multiple relationships at once)
*/
function navigator({
spec,
callbacks,
recursivePayload,
baseTableName,
}: {
readonly spec: NavigatorSpec;
// Callbacks can be modified depending on the need to make navigator versatile
readonly callbacks: NavigationCallbacks;
// Used internally to make navigator call itself multiple times
@@ -112,10 +122,10 @@ function navigator({
const childrenAreToManyElements =
relationshipIsToMany(parentRelationship) &&
!valueIsToManyIndex(parentPartName) &&
!valueIsTreeRank(parentPartName);
!valueIsTreeMeta(parentPartName);

const childrenAreRanks =
isTreeTable(table.name) && !valueIsTreeRank(parentPartName);
isTreeTable(table.name) && !valueIsTreeMeta(parentPartName);

const callbackPayload = {
table,
@@ -124,11 +134,28 @@ function navigator({

if (childrenAreToManyElements)
callbacks.handleToManyChildren(callbackPayload);
else if (childrenAreRanks) callbacks.handleTreeRanks(callbackPayload);
else if (childrenAreRanks) {
const definitions = getTreeDefinitions(table.name, 'all');

if (definitions.length > 1 && spec.useSpecificTreeInterface)
callbacks.handleTreeDefinitions(callbackPayload);
else
callbacks.handleTreeRanks({
...callbackPayload,
definitionName: spec.useSpecificTreeInterface
? definitions[0].definition.name ?? anyTreeRank
: anyTreeRank,
});
} else if (valueIsTreeDefinition(parentPartName))
callbacks.handleTreeRanks({
...callbackPayload,
definitionName: getNameFromTreeDefinitionName(parentPartName),
});
else callbacks.handleSimpleFields(callbackPayload);

const isSpecial =
valueIsToManyIndex(next.partName) || valueIsTreeRank(next.partName);
valueIsToManyIndex(next.partName) || valueIsTreeMeta(next.partName);

const nextField = isSpecial
? parentRelationship
: table.getField(next.fieldName);
@@ -141,6 +168,7 @@ function navigator({

if (typeof nextTable === 'object' && nextField?.isRelationship !== false)
navigator({
spec,
callbacks,
recursivePayload: {
table: nextTable,
@@ -285,7 +313,7 @@ export function getMappingLineData({
return {
partName: nextPart,
fieldName:
valueIsTreeRank(nextPart) || valueIsToManyIndex(nextPart)
valueIsTreeMeta(nextPart) || valueIsToManyIndex(nextPart)
? valueIsToManyIndex(mappingPath[internalState.position - 1])
? mappingPath[internalState.position - 2]
: mappingPath[internalState.position - 1]
@@ -333,14 +361,65 @@ export function getMappingLineData({
);
},

handleTreeRanks({ table }) {
handleTreeDefinitions({ table }) {
const defaultValue = getNameFromTreeDefinitionName(
internalState.defaultValue
);

commitInstanceData(
'tree',
table,
generateFieldData === 'none' ||
!hasTreeAccess(table.name as AnyTree['tableName'], 'read')
? []
: [
spec.includeAnyTreeDefinition &&
(generateFieldData === 'all' ||
internalState.defaultValue ===
formatTreeDefinition(anyTreeRank))
? [
formatTreeDefinition(anyTreeRank),
{
optionLabel: queryText.anyTree(),
isRelationship: true,
isDefault:
internalState.defaultValue ===
formatTreeDefinition(anyTreeRank),
isEnabled: true,
tableName: table.name,
},
]
: undefined,
...(spec.includeSpecificTreeRanks
? getTreeDefinitions(
table.name as AnyTree['tableName'],
'all'
).map(({ definition: { name } }) =>
name === defaultValue || generateFieldData === 'all'
? ([
formatTreeDefinition(name),
{
optionLabel: name,
isRelationship: true,
isDefault: name === defaultValue,
tableName: table.name,
},
] as const)
: undefined
)
: []),
]
);
},

handleTreeRanks({ table, definitionName }) {
const defaultValue = getNameFromTreeRankName(internalState.defaultValue);

commitInstanceData(
'tree',
table,
generateFieldData === 'none' ||
!hasTreeAccess(table.name as 'Geography', 'read')
!hasTreeAccess(table.name as AnyTree['tableName'], 'read')
? []
: [
spec.includeAnyTreeRank &&
@@ -360,26 +439,28 @@ export function getMappingLineData({
]
: undefined,
...(spec.includeSpecificTreeRanks
? getTreeDefinitions(
table.name as AnyTree['tableName'],
'all'
).flatMap(({ definition, ranks }) =>
// Exclude the root rank for each tree
ranks.slice(1).map(({ name, title }) =>
name === defaultValue || generateFieldData === 'all'
? ([
formatTreeRank(name),
{
optionLabel: title ?? name,
isRelationship: true,
isDefault: name === defaultValue,
tableName: table.name,
tableTreeDefName: definition.name,
},
] as const)
: undefined
? getTreeDefinitions(table.name as AnyTree['tableName'], 'all')
.filter(
({ definition: { name } }) =>
definitionName === name ||
definitionName === anyTreeRank
)
.flatMap(({ ranks }) =>
// Exclude the root rank for each tree
ranks.slice(1).map(({ name, title }) =>
name === defaultValue || generateFieldData === 'all'
? ([
formatTreeRank(name),
{
optionLabel: title ?? name,
isRelationship: true,
isDefault: name === defaultValue,
tableName: table.name,
},
] as const)
: undefined
)
)
)
: []),
]
);
@@ -586,6 +667,7 @@ export function getMappingLineData({
};

navigator({
spec,
callbacks,
baseTableName,
});
@@ -595,7 +677,9 @@ export function getMappingLineData({
: internalState.mappingLineData.filter(
({ customSelectSubtype }) => customSelectSubtype !== 'toMany'
);
return spec.includeAnyTreeRank || spec.includeSpecificTreeRanks
return spec.includeAnyTreeRank ||
spec.includeAnyTreeDefinition ||
spec.includeSpecificTreeRanks
? filtered
: filtered.filter(
({ customSelectSubtype }) => customSelectSubtype !== 'tree'
Original file line number Diff line number Diff line change
@@ -14,6 +14,9 @@ export type NavigatorSpec = {
// Whether no restrictions mode is enabled
readonly isNoRestrictions: () => boolean;
readonly includeToManyReferenceNumbers: boolean;
// REFACTOR: remove this attribute
readonly useSpecificTreeInterface: boolean;
readonly includeAnyTreeDefinition: boolean;
readonly includeSpecificTreeRanks: boolean;
readonly includeAnyTreeRank: boolean;
readonly includeFormattedAggregated: boolean;
@@ -40,7 +43,9 @@ const wbPlanView: NavigatorSpec = {
isNoRestrictions: () =>
userPreferences.get('workBench', 'wbPlanView', 'noRestrictionsMode'),
includeToManyReferenceNumbers: true,
useSpecificTreeInterface: true,
includeSpecificTreeRanks: true,
includeAnyTreeDefinition: false,
includeAnyTreeRank: false,
includeFormattedAggregated: false,
includeRootFormattedAggregated: false,
@@ -81,6 +86,9 @@ const queryBuilder: NavigatorSpec = {
isNoRestrictions: () =>
userPreferences.get('queryBuilder', 'general', 'noRestrictionsMode'),
includeToManyReferenceNumbers: false,
// REFACTOR: see https://github.com/specify/specify7/pull/5251
useSpecificTreeInterface: false,
includeAnyTreeDefinition: true,
includeSpecificTreeRanks: true,
includeAnyTreeRank: true,
includeFormattedAggregated: true,
@@ -111,6 +119,8 @@ const formatterEditor: NavigatorSpec = {
includeReadOnly: true,
isNoRestrictions: () => true,
includeToManyReferenceNumbers: false,
useSpecificTreeInterface: false,
includeAnyTreeDefinition: false,
includeSpecificTreeRanks: false,
includeAnyTreeRank: false,
includeFormattedAggregated: true,
@@ -133,6 +143,8 @@ const permissive: NavigatorSpec = {
includeReadOnly: true,
isNoRestrictions: () => true,
includeToManyReferenceNumbers: true,
useSpecificTreeInterface: true,
includeAnyTreeDefinition: true,
includeSpecificTreeRanks: true,
includeAnyTreeRank: true,
includeFormattedAggregated: true,
Original file line number Diff line number Diff line change
@@ -3,9 +3,14 @@ import { group, removeKey, split, toLowerCase } from '../../utils/utils';
import type { SpecifyTable } from '../DataModel/specifyTable';
import { strictGetTable } from '../DataModel/tables';
import type { Tables } from '../DataModel/types';
import { isTreeTable } from '../InitialContext/treeRanks';
import { getTreeDefinitions, isTreeTable } from '../InitialContext/treeRanks';
import { defaultColumnOptions } from './linesGetter';
import type { SplitMappingPath } from './mappingHelpers';
import {
getNameFromTreeDefinitionName,
valueIsTreeDefinition,
valueIsTreeRank,
} from './mappingHelpers';
import { getNameFromTreeRankName, valueIsToManyIndex } from './mappingHelpers';
import type {
ColumnDefinition,
@@ -33,21 +38,64 @@ const toTreeRecordRanks = (
): IR<ColumnDefinition> =>
Object.fromEntries(
rankMappedFields.map(({ mappingPath, headerName, columnOptions }) => [
mappingPath[0].toLowerCase(),
/*
* Use the last element of the mapping path to get the field name
* e.g: mappingPath when a tree is specified: ['$Kingdom', 'name'] vs when no tree is specified: ['name']
*/
mappingPath.at(-1)?.toLowerCase(),
toColumnOptions(headerName, columnOptions),
])
);

const toTreeRecordVariety = (lines: RA<SplitMappingPath>): TreeRecord => ({
ranks: Object.fromEntries(
indexMappings(lines).map(([fullRankName, rankMappedFields]) => [
getNameFromTreeRankName(fullRankName),
{
treeNodeCols: toTreeRecordRanks(rankMappedFields),
},
])
),
});
const toTreeRecordVariety = (lines: RA<SplitMappingPath>): TreeRecord => {
const treeDefinitions = getTreeDefinitions('Taxon', 'all');

return {
ranks: Object.fromEntries(
indexMappings(lines).flatMap(([fullName, rankMappedFields]) => {
/*
* For collections with only 1 tree, rankMappedFields already has the correct format for generating an upload plan
* When there are multiple trees in the mapper, rankMappedFields is actually a mapping path of trees mapped to ranks
* which means we need to get the index mappings of it again to get the actual rankMappedFields
*/
const rankMappings: RA<readonly [string, RA<SplitMappingPath>]> =
valueIsTreeRank(fullName)
? [[fullName, rankMappedFields]]
: indexMappings(rankMappedFields);

const treeName: string | undefined = valueIsTreeDefinition(fullName)
? getNameFromTreeDefinitionName(fullName)
: undefined;

return rankMappings.map(([fullName, mappedFields]) => {
const rankName = getNameFromTreeRankName(fullName);

// Resolve treeId using treeName or using rankName as fallback
const treeId =
treeName === undefined
? treeDefinitions.find(({ ranks }) =>
ranks.find((r) => r.name === rankName)
)?.definition.id
: treeDefinitions.find(
({ definition }) => definition.name === treeName
)?.definition.id;

return [
treeId === undefined
? rankName
: `${
treeName === undefined ? '' : treeName + RANK_KEY_DELIMITER
}${rankName}`,
{
treeNodeCols: toTreeRecordRanks(mappedFields),
treeId,
},
];
});
})
),
};
};

function toUploadTable(
table: SpecifyTable,
@@ -165,3 +213,6 @@ const indexMappings = (
] as const
)
);

// Delimiter used for rank name keys i.e: <rankname>~><treeId>
export const RANK_KEY_DELIMITER = '~>';
Original file line number Diff line number Diff line change
@@ -3,10 +3,13 @@ import type { SpecifyTable } from '../DataModel/specifyTable';
import { strictGetTable } from '../DataModel/tables';
import type { Tables } from '../DataModel/types';
import { softFail } from '../Errors/Crash';
import { getTreeDefinitions } from '../InitialContext/treeRanks';
import { defaultColumnOptions } from './linesGetter';
import type { MappingPath } from './Mapper';
import type { SplitMappingPath } from './mappingHelpers';
import { formatTreeDefinition } from './mappingHelpers';
import { formatToManyIndex, formatTreeRank } from './mappingHelpers';
import { RANK_KEY_DELIMITER } from './uploadPlanBuilder';

export type MatchBehaviors = 'ignoreAlways' | 'ignoreNever' | 'ignoreWhenBlank';

@@ -34,7 +37,10 @@ type UploadTableVariety =
| { readonly uploadTable: UploadTable };

export type TreeRecord = {
readonly ranks: IR<string | { readonly treeNodeCols: IR<ColumnDefinition> }>;
readonly ranks: IR<
| string
| { readonly treeNodeCols: IR<ColumnDefinition>; readonly treeId?: number }
>;
};

type TreeRecordVariety =
@@ -70,10 +76,23 @@ const parseTree = (
name: rankData,
}
: rankData.treeNodeCols,
[...mappingPath, formatTreeRank(rankName)]
[
...mappingPath,
...(typeof rankData === 'object' &&
typeof rankData.treeId === 'number' &&
getTreeDefinitions('Taxon', 'all').length > 1
? [resolveTreeId(rankData.treeId)]
: []),
formatTreeRank(getRankNameWithoutTreeId(rankName)),
]
)
);

const resolveTreeId = (id: number): string => {
const treeDefinition = getTreeDefinitions('Taxon', id);
return formatTreeDefinition(treeDefinition[0].definition.name);
};

function parseTreeTypes(
table: SpecifyTable,
uploadPlan: TreeRecordVariety,
@@ -194,3 +213,11 @@ export function parseUploadPlan(uploadPlan: UploadPlan): {
),
};
}

/**
* Returns the tree rank name after stripping its tree id: (e.x Kingdom~>1 => Kingdom)
* NOTE: Does not consider whether rankName is formatted with $ or not.
* Opposite of uploadPlanBuilder.ts > formatTreeRankWithTreeId()
*/
const getRankNameWithoutTreeId = (rankName: string): string =>
rankName.split(RANK_KEY_DELIMITER)[0];
3 changes: 3 additions & 0 deletions specifyweb/frontend/js_src/lib/localization/query.ts
Original file line number Diff line number Diff line change
@@ -343,6 +343,9 @@ export const queryText = createDictionary({
'uk-ua': '(будь-який ранг)',
'de-ch': '(jeder Rang)',
},
anyTree: {
'en-us': '(any tree)',
},
moveUp: {
comment: 'As in move it up',
'en-us': 'Move Up',
18 changes: 12 additions & 6 deletions specifyweb/frontend/js_src/lib/tests/fixtures/uploadplan.1.json
Original file line number Diff line number Diff line change
@@ -283,34 +283,40 @@
"default": null,
"column": "Class"
}
}
},
"treeId": 1
},
"Family": {
"treeNodeCols": {
"name": "Family"
}
},
"treeId": 1
},
"Genus": {
"treeNodeCols": {
"name": "Genus"
}
},
"treeId": 1
},
"Subgenus": {
"treeNodeCols": {
"name": "Subgenus"
}
},
"treeId": 1
},
"Species": {
"treeNodeCols": {
"name": "Species",
"author": "Species Author"
}
},
"treeId": 1
},
"Subspecies": {
"treeNodeCols": {
"name": "Subspecies",
"author": "Subspecies Author"
}
},
"treeId": 1
}
}
}
2 changes: 2 additions & 0 deletions specifyweb/specify/tree_utils.py
Original file line number Diff line number Diff line change
@@ -5,6 +5,8 @@

lookup = lambda tree: (tree.lower() + 'treedef')

SPECIFY_TREES = {"taxon", "storage", "geography", "geologictimeperiod", "lithostrat"}

def get_search_filters(collection: spmodels.Collection, tree: str):
tree_name = tree.lower()
if tree_name == 'storage':
50 changes: 32 additions & 18 deletions specifyweb/workbench/tasks.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
from typing import Optional, Any

from celery import shared_task # type: ignore
from celery import Task # type: ignore
from celery.utils.log import get_task_logger # type: ignore

from django.db import connection, transaction
@@ -14,37 +15,50 @@

logger = get_task_logger(__name__)

@app.task(base=LogErrorsTask, bind=True)
def upload(self, collection_id: int, uploading_agent_id: int, ds_id: int, no_commit: bool, allow_partial: bool) -> None:

def progress(current: int, total: Optional[int]) -> None:
if not self.request.called_directly:
self.update_state(state='PROGRESS', meta={'current': current, 'total': total})

def upload_data(
collection_id: int,
uploading_agent_id: int,
ds_id: int,
no_commit: bool,
allow_partial: bool,
task: Optional[Task]=None,
progress=None,
) -> None:
with transaction.atomic():
ds = Spdataset.objects.select_for_update().get(id=ds_id)
collection = Collection.objects.get(id=collection_id)

if ds.uploaderstatus is None:
logger.info("dataset is not assigned to an upload task")
return
if task is not None:
if ds.uploaderstatus is None:
logger.info("dataset is not assigned to an upload task")
return

if ds.uploaderstatus['taskid'] != self.request.id:
logger.info("dataset is not assigned to this task")
return
if ds.uploaderstatus['taskid'] != task.request.id:
logger.info("dataset is not assigned to this task")
return

if not (ds.uploaderstatus['operation'] == ("validating" if no_commit else "uploading")): raise AssertionError(
f"Invalid status '{ds.uploaderstatus['operation']}' for upload. Expected 'validating' or 'uploading'",
{"uploadStatus" : ds.uploaderstatus['operation'],
"operation" : "upload",
"expectedUploadStatus" : 'validating , uploading',
"localizationKey" : "invalidUploadStatus"})
if not (ds.uploaderstatus['operation'] == ("validating" if no_commit else "uploading")): raise AssertionError(
f"Invalid status '{ds.uploaderstatus['operation']}' for upload. Expected 'validating' or 'uploading'",
{"uploadStatus" : ds.uploaderstatus['operation'],
"operation" : "upload",
"expectedUploadStatus" : 'validating , uploading',
"localizationKey" : "invalidUploadStatus"})

do_upload_dataset(collection, uploading_agent_id, ds, no_commit, allow_partial, progress)

ds.uploaderstatus = None
ds.save(update_fields=['uploaderstatus'])

@app.task(base=LogErrorsTask, bind=True)
def upload(self, collection_id: int, uploading_agent_id: int, ds_id: int, no_commit: bool, allow_partial: bool) -> None:

def progress(current: int, total: Optional[int]) -> None:
if not self.request.called_directly:
self.update_state(state='PROGRESS', meta={'current': current, 'total': total})

upload_data(collection_id, uploading_agent_id, ds_id, no_commit, allow_partial, self, progress)

@app.task(base=LogErrorsTask, bind=True)
def unupload(self, ds_id: int, agent_id: int) -> None:

31 changes: 24 additions & 7 deletions specifyweb/workbench/upload/scoping.py
Original file line number Diff line number Diff line change
@@ -9,7 +9,7 @@
from .uploadable import Uploadable, ScopedUploadable
from .upload_table import UploadTable, DeferredScopeUploadTable, ScopedUploadTable, OneToOneTable, ScopedOneToOneTable
from .tomany import ToManyRecord, ScopedToManyRecord
from .treerecord import TreeRecord, ScopedTreeRecord
from .treerecord import TreeRank, TreeRankRecord, TreeRecord, ScopedTreeRecord
from .column_options import ColumnOptions, ExtendedColumnOptions

""" There are cases in which the scoping of records should be dependent on another record/column in a WorkBench dataset.
@@ -186,10 +186,10 @@ def apply_scoping_to_tomanyrecord(tmr: ToManyRecord, collection) -> ScopedToMany
def apply_scoping_to_treerecord(tr: TreeRecord, collection) -> ScopedTreeRecord:
table = datamodel.get_table_strict(tr.name)

treedef = None
if table.name == 'Taxon':
# TODO: Add scoping in the workbench via the Upload Plan
# treedef = get_taxon_treedef(collection)
treedef = collection.discipline.taxontreedef
if treedef is None:
treedef = collection.discipline.taxontreedef

elif table.name == 'Geography':
treedef = collection.discipline.geographytreedef
@@ -206,19 +206,36 @@ def apply_scoping_to_treerecord(tr: TreeRecord, collection) -> ScopedTreeRecord:
else:
raise Exception(f'unexpected tree type: {table.name}')

if treedef is None:
raise ValueError(f"Could not find treedef for table {table.name}")

treedefitems = list(treedef.treedefitems.order_by('rankid'))
treedef_ranks = [tdi.name for tdi in treedefitems]
for rank in tr.ranks:
if rank not in treedef_ranks:
if rank not in treedef_ranks and isinstance(rank, TreeRankRecord) and not rank.check_rank(table.name):
raise Exception(f'"{rank}" not among {table.name} tree ranks: {treedef_ranks}')

root = list(getattr(models, table.name.capitalize()).objects.filter(definitionitem=treedefitems[0])[:1]) # assume there is only one

scoped_ranks: Dict[TreeRankRecord, Dict[str, ExtendedColumnOptions]] = {
(
TreeRank.create(
r, table.name, treedef.id if treedef else None
).tree_rank_record()
if isinstance(r, str)
else r
): {
f: extend_columnoptions(colopts, collection, table.name, f)
for f, colopts in cols.items()
}
for r, cols in tr.ranks.items()
}

return ScopedTreeRecord(
name=tr.name,
ranks={r: {f: extend_columnoptions(colopts, collection, table.name, f) for f, colopts in cols.items()} for r, cols in tr.ranks.items()},
ranks=scoped_ranks,
treedef=treedef,
treedefitems=list(treedef.treedefitems.order_by('rankid')),
treedefitems=treedefitems,
root=root[0] if root else None,
disambiguation={},
)
10 changes: 10 additions & 0 deletions specifyweb/workbench/upload/tests/testschema.py
Original file line number Diff line number Diff line change
@@ -14,10 +14,19 @@
from .base import UploadTestsBase
from . import example_plan

def set_plan_treeId():
# Set the treeId in example_plan.json dynamically, so that the unit test doesn't depend on the static treeId to always be the same.
from specifyweb.specify.models import Taxontreedefitem
tree_id = Taxontreedefitem.objects.filter(name='Species').first().treedef_id
example_plan_ranks = example_plan.json['uploadable']['uploadTable']['toMany']['determinations'][0]['toOne']['taxon']['treeRecord']['ranks']
for rank in ['Species', 'Subspecies']:
example_plan_ranks[rank]['treeId'] = tree_id

class SchemaTests(UploadTestsBase):
maxDiff = None

def test_schema_parsing(self) -> None:
set_plan_treeId()
Draft7Validator.check_schema(schema)
validate(example_plan.json, schema)
plan = parse_plan(self.collection, example_plan.json).apply_scoping(self.collection)
@@ -26,6 +35,7 @@ def test_schema_parsing(self) -> None:
self.assertEqual(repr(plan), repr(self.example_plan))

def test_unparsing(self) -> None:
set_plan_treeId()
self.assertEqual(example_plan.json, parse_plan(self.collection, example_plan.json).unparse())

def test_reject_internal_tree_columns(self) -> None:
446 changes: 424 additions & 22 deletions specifyweb/workbench/upload/treerecord.py

Large diffs are not rendered by default.

64 changes: 51 additions & 13 deletions specifyweb/workbench/upload/upload_plan_schema.py
Original file line number Diff line number Diff line change
@@ -1,16 +1,18 @@
from typing import Dict, Any, Union, Tuple
from functools import reduce
from os import name
from typing import Dict, Any, Optional, Union, Tuple
import logging

from specifyweb.specify.datamodel import datamodel, Table, Relationship
from specifyweb.specify.load_datamodel import DoesNotExistError
from specifyweb.specify import models

from .upload_table import DeferredScopeUploadTable, UploadTable, OneToOneTable, MustMatchTable
from .tomany import ToManyRecord
from .treerecord import TreeRecord, MustMatchTreeRecord
from .treerecord import TreeRank, TreeRankRecord, TreeRecord, MustMatchTreeRecord
from .uploadable import Uploadable
from .column_options import ColumnOptions
from .scoping import DEFERRED_SCOPING

logger = logging.getLogger(__name__)

schema: Dict = {
'title': 'Specify 7 Workbench Upload Plan',
@@ -33,7 +35,7 @@
'additionalProperties': False
},
]
},
}
},
'required': [ 'baseTableName', 'uploadable' ],
'additionalProperties': False,
@@ -91,6 +93,7 @@
'type': 'object',
'properties': {
'treeNodeCols': { '$ref': '#/definitions/treeNodeCols' },
'treeId': { '$ref': '#definitions/treeId'}
},
'required': [ 'treeNodeCols' ],
'additionalProperties': False
@@ -140,6 +143,13 @@
],
},

'treeId': {
"oneOf": [
{ "type": "integer" },
{ "type": "null" }
]
},

'wbcols': {
'type': 'object',
'description': 'Maps the columns of the destination table to the headers of the source columns of input data.',
@@ -288,17 +298,45 @@ def defer_scope_upload_table(default_collection, table: Table, to_parse: Dict, d
)

def parse_tree_record(collection, table: Table, to_parse: Dict) -> TreeRecord:
ranks = {
rank: {'name': parse_column_options(name_or_cols)} if isinstance(name_or_cols, str)
else {k: parse_column_options(v) for k,v in name_or_cols['treeNodeCols'].items() }
for rank, name_or_cols in to_parse['ranks'].items()
}
for rank, cols in ranks.items():
assert 'name' in cols, to_parse
"""Parse tree record from the given data"""

def parse_rank(name_or_cols):
if isinstance(name_or_cols, str):
return name_or_cols, {'name': parse_column_options(name_or_cols)}, None
tree_node_cols = {k: parse_column_options(v) for k, v in name_or_cols['treeNodeCols'].items()}
rank_name = name_or_cols['treeNodeCols']['name']
treedefid = name_or_cols.get('treeId')
return rank_name, tree_node_cols, treedefid

def create_tree_rank_record(rank, table_name, treedefid):
return TreeRank.create(rank, table_name, treedefid).tree_rank_record()

def parse_ranks(to_parse, table_name):
def parse_single_rank(rank, name_or_cols):
rank_name, parsed_cols, treedefid = parse_rank(name_or_cols)
tree_rank_record = create_tree_rank_record(rank, table_name, treedefid)
return tree_rank_record, parsed_cols

def aggregate_ranks(acc, item):
rank, name_or_cols = item
tree_rank_record, parsed_cols = parse_single_rank(rank, name_or_cols)
acc[tree_rank_record] = parsed_cols
return acc

ranks = reduce(aggregate_ranks, to_parse['ranks'].items(), {})
return ranks

# Validate ranks by checking if 'name' is present in each rank
def validate_ranks(ranks, to_parse):
for rank, cols in ranks.items():
assert 'name' in cols, to_parse

ranks = parse_ranks(to_parse, table.name)
validate_ranks(ranks, to_parse)

return TreeRecord(
name=table.django_name,
ranks=ranks,
ranks=ranks
)

def parse_to_many_record(collection, table: Table, to_parse: Dict) -> ToManyRecord:
50 changes: 32 additions & 18 deletions specifyweb/workbench/views.py
Original file line number Diff line number Diff line change
@@ -480,8 +480,9 @@ def dataset(request, ds: models.Spdataset) -> http.HttpResponse:
# TODO fix this
# return http.HttpResponse(f"upload plan is invalid: {e}", status=400)
pass

new_cols = upload_plan_schema.parse_plan(request.specify_collection, plan).get_cols() - set(ds.columns)

parsed_plan = upload_plan_schema.parse_plan(request.specify_collection, plan)
new_cols = parsed_plan.get_cols() - set(ds.columns)
if new_cols:
ncols = len(ds.columns)
ds.columns += list(new_cols)
@@ -626,22 +627,35 @@ def upload(request, ds, no_commit: bool, allow_partial: bool) -> http.HttpRespon
if ds.was_uploaded():
return http.HttpResponse('dataset has already been uploaded.', status=400)

taskid = str(uuid4())
async_result = tasks.upload.apply_async([
request.specify_collection.id,
request.specify_user_agent.id,
ds.id,
no_commit,
allow_partial
], task_id=taskid)
ds.uploaderstatus = {
'operation': "validating" if no_commit else "uploading",
'taskid': taskid
}
ds.save(update_fields=['uploaderstatus'])

return http.JsonResponse(async_result.id, safe=False)

data = json.loads(request.body) if request.body else {}
background = True
if 'background' in data:
background = bool(data['background']) # {"background": false}

if background:
taskid = str(uuid4())
async_result = tasks.upload.apply_async([
request.specify_collection.id,
request.specify_user_agent.id,
ds.id,
no_commit,
allow_partial
], task_id=taskid)
ds.uploaderstatus = {
'operation': "validating" if no_commit else "uploading",
'taskid': taskid
}
ds.save(update_fields=['uploaderstatus'])
return http.JsonResponse(async_result.id, safe=False)
else:
tasks.upload_data(
request.specify_collection.id,
request.specify_user_agent.id,
ds.id,
no_commit,
allow_partial,
)
return http.JsonResponse(None, safe=False)

@openapi(schema={
'post': {

0 comments on commit dfa30a6

Please sign in to comment.