diff --git a/specifyweb/frontend/js_src/lib/components/DataModel/__tests__/schemaBase.test.ts b/specifyweb/frontend/js_src/lib/components/DataModel/__tests__/schemaBase.test.ts index 8e16373ae35..78524afc219 100644 --- a/specifyweb/frontend/js_src/lib/components/DataModel/__tests__/schemaBase.test.ts +++ b/specifyweb/frontend/js_src/lib/components/DataModel/__tests__/schemaBase.test.ts @@ -24,5 +24,6 @@ test('domain data is fetched and parsed correctly', async () => ], paleoContextChildTable: 'collectionobject', referenceSymbol: '#', - treeSymbol: '$', + treeRankSymbol: '$', + treeDefinitionSymbol: '%', })); diff --git a/specifyweb/frontend/js_src/lib/components/DataModel/schema.ts b/specifyweb/frontend/js_src/lib/components/DataModel/schema.ts index f1ed868f49c..a121851d82c 100644 --- a/specifyweb/frontend/js_src/lib/components/DataModel/schema.ts +++ b/specifyweb/frontend/js_src/lib/components/DataModel/schema.ts @@ -29,7 +29,8 @@ type Schema = { 'Institution' ]; readonly referenceSymbol: string; - readonly treeSymbol: string; + readonly treeDefinitionSymbol: string; + readonly treeRankSymbol: string; readonly fieldPartSeparator: string; }; @@ -60,8 +61,10 @@ const schemaBase: Writable = { // Prefix for -to-many indexes referenceSymbol: '#', + // Prefix for Tree Definitions + treeDefinitionSymbol: '%', // Prefix for tree ranks - treeSymbol: '$', + treeRankSymbol: '$', // Separator for partial fields (date parts in Query Builder) fieldPartSeparator: '-', }; diff --git a/specifyweb/frontend/js_src/lib/components/InitialContext/treeRanks.ts b/specifyweb/frontend/js_src/lib/components/InitialContext/treeRanks.ts index 44c6deeba2b..29c84c9da1c 100644 --- a/specifyweb/frontend/js_src/lib/components/InitialContext/treeRanks.ts +++ b/specifyweb/frontend/js_src/lib/components/InitialContext/treeRanks.ts @@ -118,11 +118,14 @@ export function getTreeDefinitions( tableName ); - return typeof treeDefinitionId === 'number' - ? specificTreeDefinitions.filter( - ({ definition }) => definition.id === treeDefinitionId - ) - : specificTreeDefinitions; + if (typeof treeDefinitionId === 'number') { + const resolvedDefinition = specificTreeDefinitions.find( + ({ definition }) => definition.id === treeDefinitionId + ); + return resolvedDefinition === undefined + ? specificTreeDefinitions + : [resolvedDefinition]; + } else return specificTreeDefinitions; } export function getTreeDefinitionItems( diff --git a/specifyweb/frontend/js_src/lib/components/Syncer/__tests__/index.test.ts b/specifyweb/frontend/js_src/lib/components/Syncer/__tests__/index.test.ts index 89b362c28c0..bb7abf22a9c 100644 --- a/specifyweb/frontend/js_src/lib/components/Syncer/__tests__/index.test.ts +++ b/specifyweb/frontend/js_src/lib/components/Syncer/__tests__/index.test.ts @@ -94,8 +94,8 @@ test('Editing Data Object Formatter', () => { - - + + " `); diff --git a/specifyweb/frontend/js_src/lib/components/WbPlanView/LineComponents.tsx b/specifyweb/frontend/js_src/lib/components/WbPlanView/LineComponents.tsx index df5aceaf43f..433a4761b77 100644 --- a/specifyweb/frontend/js_src/lib/components/WbPlanView/LineComponents.tsx +++ b/specifyweb/frontend/js_src/lib/components/WbPlanView/LineComponents.tsx @@ -37,7 +37,6 @@ export type HtmlGeneratorFieldData = { readonly isDefault?: boolean; readonly isRelationship?: boolean; readonly tableName?: keyof Tables; - readonly tableTreeDefName?: string; }; type MappingLineBaseProps = { diff --git a/specifyweb/frontend/js_src/lib/components/WbPlanView/__tests__/mappingHelpers.test.ts b/specifyweb/frontend/js_src/lib/components/WbPlanView/__tests__/mappingHelpers.test.ts index d6df35e6dfb..866f9ec1520 100644 --- a/specifyweb/frontend/js_src/lib/components/WbPlanView/__tests__/mappingHelpers.test.ts +++ b/specifyweb/frontend/js_src/lib/components/WbPlanView/__tests__/mappingHelpers.test.ts @@ -16,7 +16,7 @@ theories(valueIsToManyIndex, [ [[`${schema.referenceSymbol}2`], true], [[`${schema.referenceSymbol}999`], true], [['collectionobject'], false], - [[`${schema.treeSymbol}Kingdom`], false], + [[`${schema.treeRankSymbol}Kingdom`], false], ]); theories(valueIsTreeRank, [ @@ -24,8 +24,8 @@ theories(valueIsTreeRank, [ [[`${schema.referenceSymbol}2`], false], [[`${schema.referenceSymbol}999`], false], [['collectionobject'], false], - [[`${schema.treeSymbol}Kingdom`], true], - [[`${schema.treeSymbol}County`], true], + [[`${schema.treeRankSymbol}Kingdom`], true], + [[`${schema.treeRankSymbol}County`], true], ]); theories(getNumberFromToManyIndex, [ @@ -36,8 +36,8 @@ theories(getNumberFromToManyIndex, [ ]); theories(getNameFromTreeRankName, [ - [[`${schema.treeSymbol}Kingdom`], 'Kingdom'], - [[`${schema.treeSymbol}County`], 'County'], + [[`${schema.treeRankSymbol}Kingdom`], 'Kingdom'], + [[`${schema.treeRankSymbol}County`], 'County'], ]); theories(findDuplicateMappings, [ diff --git a/specifyweb/frontend/js_src/lib/components/WbPlanView/__tests__/navigator.test.ts b/specifyweb/frontend/js_src/lib/components/WbPlanView/__tests__/navigator.test.ts index 7ebc8389786..cec41085ea3 100644 --- a/specifyweb/frontend/js_src/lib/components/WbPlanView/__tests__/navigator.test.ts +++ b/specifyweb/frontend/js_src/lib/components/WbPlanView/__tests__/navigator.test.ts @@ -314,70 +314,60 @@ theories(getMappingLineData, [ isRelationship: true, isDefault: false, tableName: 'Taxon', - tableTreeDefName: 'Taxonomy', }, $Phylum: { optionLabel: 'Phylum', isRelationship: true, isDefault: false, tableName: 'Taxon', - tableTreeDefName: 'Taxonomy', }, $Class: { optionLabel: 'Class', isRelationship: true, isDefault: false, tableName: 'Taxon', - tableTreeDefName: 'Taxonomy', }, $Order: { optionLabel: 'Order', isRelationship: true, isDefault: false, tableName: 'Taxon', - tableTreeDefName: 'Taxonomy', }, $Family: { optionLabel: 'Family', isRelationship: true, isDefault: true, tableName: 'Taxon', - tableTreeDefName: 'Taxonomy', }, $Subfamily: { optionLabel: 'Subfamily', isRelationship: true, isDefault: false, tableName: 'Taxon', - tableTreeDefName: 'Taxonomy', }, $Genus: { optionLabel: 'Genus', isRelationship: true, isDefault: false, tableName: 'Taxon', - tableTreeDefName: 'Taxonomy', }, $Subgenus: { optionLabel: 'Subgenus', isRelationship: true, isDefault: false, tableName: 'Taxon', - tableTreeDefName: 'Taxonomy', }, $Species: { optionLabel: 'Species', isRelationship: true, isDefault: false, tableName: 'Taxon', - tableTreeDefName: 'Taxonomy', }, $Subspecies: { optionLabel: 'Subspecies', isRelationship: true, isDefault: false, tableName: 'Taxon', - tableTreeDefName: 'Taxonomy', }, }, tableName: 'Taxon', @@ -870,70 +860,60 @@ theories(getMappingLineData, [ isRelationship: true, optionLabel: 'Class', tableName: 'Taxon', - tableTreeDefName: 'Taxonomy', }, $Family: { isDefault: true, isRelationship: true, optionLabel: 'Family', tableName: 'Taxon', - tableTreeDefName: 'Taxonomy', }, $Genus: { isDefault: false, isRelationship: true, optionLabel: 'Genus', tableName: 'Taxon', - tableTreeDefName: 'Taxonomy', }, $Kingdom: { isDefault: false, isRelationship: true, optionLabel: 'Kingdom', tableName: 'Taxon', - tableTreeDefName: 'Taxonomy', }, $Order: { isDefault: false, isRelationship: true, optionLabel: 'Order', tableName: 'Taxon', - tableTreeDefName: 'Taxonomy', }, $Phylum: { isDefault: false, isRelationship: true, optionLabel: 'Phylum', tableName: 'Taxon', - tableTreeDefName: 'Taxonomy', }, $Species: { isDefault: false, isRelationship: true, optionLabel: 'Species', tableName: 'Taxon', - tableTreeDefName: 'Taxonomy', }, $Subfamily: { isDefault: false, isRelationship: true, optionLabel: 'Subfamily', tableName: 'Taxon', - tableTreeDefName: 'Taxonomy', }, $Subgenus: { isDefault: false, isRelationship: true, optionLabel: 'Subgenus', tableName: 'Taxon', - tableTreeDefName: 'Taxonomy', }, $Subspecies: { isDefault: false, isRelationship: true, optionLabel: 'Subspecies', tableName: 'Taxon', - tableTreeDefName: 'Taxonomy', }, }, selectLabel: localized('Taxon'), diff --git a/specifyweb/frontend/js_src/lib/components/WbPlanView/mappingHelpers.ts b/specifyweb/frontend/js_src/lib/components/WbPlanView/mappingHelpers.ts index a5af31c19c9..dc65961182d 100644 --- a/specifyweb/frontend/js_src/lib/components/WbPlanView/mappingHelpers.ts +++ b/specifyweb/frontend/js_src/lib/components/WbPlanView/mappingHelpers.ts @@ -29,9 +29,20 @@ export const valueIsToManyIndex = (value: string | undefined): boolean => value?.slice(0, schema.referenceSymbol.length) === schema.referenceSymbol || false; +/** + * Returns whether a value is any special tree-related meta information + * (e.g. tree defintiion, tree rank) + */ +export const valueIsTreeMeta = (value: string | undefined): boolean => + valueIsTreeDefinition(value) || valueIsTreeRank(value); + +/** Returns whether a value is a tree definition name (e.x %Taxonomy) */ +export const valueIsTreeDefinition = (value: string | undefined): boolean => + value?.startsWith(schema.treeDefinitionSymbol) ?? false; + /** Returns whether a value is a tree rank name (e.x $Kingdom, $Order) */ export const valueIsTreeRank = (value: string | undefined): boolean => - value?.startsWith(schema.treeSymbol) || false; + value?.startsWith(schema.treeRankSymbol) ?? false; /** * Returns index from a formatted -to-many index value (e.x #1 => 1) @@ -40,6 +51,14 @@ export const valueIsTreeRank = (value: string | undefined): boolean => export const getNumberFromToManyIndex = (value: string): number => Number(value.slice(schema.referenceSymbol.length)); +/** + * Returns tree definition name from a complete tree definition name + * (e.x %Taxonomy => Taxonomy) + * Opposite of formatTreeDefinition + */ +export const getNameFromTreeDefinitionName = (value: string): string => + value.slice(schema.treeDefinitionSymbol.length); + /* * BUG: in places where output of this function is displayed to the user, * make sure to use tree rank title instead of name @@ -51,7 +70,7 @@ export const getNumberFromToManyIndex = (value: string): number => * */ export const getNameFromTreeRankName = (value: string): string => - value.slice(schema.treeSymbol.length); + value.slice(schema.treeRankSymbol.length); /** * Returns a formatted -to-many index from an index (e.x 1 => #1) @@ -61,7 +80,7 @@ export const formatToManyIndex = (index: number): string => `${schema.referenceSymbol}${index}`; // Meta fields -export const anyTreeRank = `${schema.fieldPartSeparator}any`; +export const anyTreeRank = `${schema.fieldPartSeparator}any` as const; export const formattedEntry = `${schema.fieldPartSeparator}formatted`; /** @@ -75,6 +94,14 @@ export const formattedEntry = `${schema.fieldPartSeparator}formatted`; */ export const emptyMapping = '0'; +/** + * Returns a complete tree definition name from a tree definition name + * (e.x Taxononomy => %Taxonomy) + * Opposite of getNameFromTreeDefinitionName + */ +export const formatTreeDefinition = (definitionName: string): string => + `${schema.treeDefinitionSymbol}${definitionName}`; + /** * Returns a complete tree rank name from a tree rank name * (e.x Kingdom => $Kingdom) @@ -82,7 +109,7 @@ export const emptyMapping = '0'; * */ export const formatTreeRank = (rankName: string): string => - `${schema.treeSymbol}${rankName}`; + `${schema.treeRankSymbol}${rankName}`; // Match fields names like startDate_fullDate, but not _formatted export const valueIsPartialField = (value: string): boolean => @@ -160,7 +187,7 @@ export const getGenericMappingPath = (mappingPath: MappingPath): MappingPath => mappingPath.filter( (mappingPathPart) => !valueIsToManyIndex(mappingPathPart) && - !valueIsTreeRank(mappingPathPart) && + !valueIsTreeMeta(mappingPathPart) && mappingPathPart !== formattedEntry && mappingPathPart !== emptyMapping ); diff --git a/specifyweb/frontend/js_src/lib/components/WbPlanView/mappingPreview.ts b/specifyweb/frontend/js_src/lib/components/WbPlanView/mappingPreview.ts index 3a27a0fade9..ad03b945177 100644 --- a/specifyweb/frontend/js_src/lib/components/WbPlanView/mappingPreview.ts +++ b/specifyweb/frontend/js_src/lib/components/WbPlanView/mappingPreview.ts @@ -22,6 +22,7 @@ import { parsePartialField, valueIsPartialField, valueIsToManyIndex, + valueIsTreeDefinition, valueIsTreeRank, } from './mappingHelpers'; import { getMappingLineData } from './navigator'; @@ -120,8 +121,11 @@ export function generateMappingPathPreview( : 1; const toManyIndexFormatted = toManyIndexNumber > 1 ? toManyIndex : undefined; - const [databaseFieldName, databaseTableOrRankName, databaseParentTableName] = - mappingPathSubset([baseTableName, ...mappingPath]); + const [ + databaseFieldName, + databaseTableOrRankName, + databaseParentTableOrTreeName, + ] = mappingPathSubset([baseTableName, ...mappingPath]); // Attributes parts of filedLables to each variable or creates one if empty const [ @@ -129,7 +133,7 @@ export function generateMappingPathPreview( tableOrRankName = camelToHuman( getNameFromTreeRankName(databaseTableOrRankName) ), - parentTableName = camelToHuman(databaseParentTableName), + parentTableOrTreeName = camelToHuman(databaseParentTableOrTreeName), ] = mappingPathSubset(fieldLabels); const isAnyRank = databaseTableOrRankName === formatTreeRank(anyTreeRank); @@ -163,16 +167,20 @@ export function generateMappingPathPreview( const tableNameFormatted = tablesToHide.has(databaseTableOrRankName) && databaseFieldName !== formattedEntry - ? [parentTableName || tableNameNonEmpty] + ? [parentTableOrTreeName || tableNameNonEmpty] : genericTables.has(databaseTableOrRankName) - ? [parentTableName, tableNameNonEmpty] + ? [parentTableOrTreeName, tableNameNonEmpty] : [tableNameNonEmpty]; return filterArray([ ...(valueIsTreeRank(databaseTableOrRankName) - ? [isAnyRank ? parentTableName : tableOrRankName] + ? [isAnyRank ? parentTableOrTreeName : tableOrRankName] : tableNameFormatted), fieldNameFormatted, + ...(valueIsTreeRank(databaseTableOrRankName) && + valueIsTreeDefinition(databaseParentTableOrTreeName) + ? [parentTableOrTreeName] + : []), toManyIndexFormatted, ]) .filter(Boolean) diff --git a/specifyweb/frontend/js_src/lib/components/WbPlanView/navigator.ts b/specifyweb/frontend/js_src/lib/components/WbPlanView/navigator.ts index f3129cc3a72..d551d1e996e 100644 --- a/specifyweb/frontend/js_src/lib/components/WbPlanView/navigator.ts +++ b/specifyweb/frontend/js_src/lib/components/WbPlanView/navigator.ts @@ -29,14 +29,17 @@ import { formatPartialField, formattedEntry, formatToManyIndex, + formatTreeDefinition, formatTreeRank, getGenericMappingPath, + getNameFromTreeDefinitionName, getNameFromTreeRankName, parsePartialField, relationshipIsToMany, valueIsPartialField, valueIsToManyIndex, - valueIsTreeRank, + valueIsTreeDefinition, + valueIsTreeMeta, } from './mappingHelpers'; import { getMaxToManyIndex, isCircularRelationship } from './modelHelpers'; import type { NavigatorSpec } from './navigatorSpecs'; @@ -64,9 +67,14 @@ type NavigationCallbacks = { readonly handleToManyChildren: ( callbackPayload: Readonly ) => void; + readonly handleTreeDefinitions: ( + callbackPayload: Readonly + ) => void; // Handles tree ranks children readonly handleTreeRanks: ( - callbackPayload: Readonly + callbackPayload: Readonly & { + readonly definitionName: string; + } ) => void; // Handles field and relationships readonly handleSimpleFields: ( @@ -84,10 +92,12 @@ type NavigationCallbacks = { * fields from multiple relationships at once) */ function navigator({ + spec, callbacks, recursivePayload, baseTableName, }: { + readonly spec: NavigatorSpec; // Callbacks can be modified depending on the need to make navigator versatile readonly callbacks: NavigationCallbacks; // Used internally to make navigator call itself multiple times @@ -112,10 +122,10 @@ function navigator({ const childrenAreToManyElements = relationshipIsToMany(parentRelationship) && !valueIsToManyIndex(parentPartName) && - !valueIsTreeRank(parentPartName); + !valueIsTreeMeta(parentPartName); const childrenAreRanks = - isTreeTable(table.name) && !valueIsTreeRank(parentPartName); + isTreeTable(table.name) && !valueIsTreeMeta(parentPartName); const callbackPayload = { table, @@ -124,11 +134,28 @@ function navigator({ if (childrenAreToManyElements) callbacks.handleToManyChildren(callbackPayload); - else if (childrenAreRanks) callbacks.handleTreeRanks(callbackPayload); + else if (childrenAreRanks) { + const definitions = getTreeDefinitions(table.name, 'all'); + + if (definitions.length > 1 && spec.useSpecificTreeInterface) + callbacks.handleTreeDefinitions(callbackPayload); + else + callbacks.handleTreeRanks({ + ...callbackPayload, + definitionName: spec.useSpecificTreeInterface + ? definitions[0].definition.name ?? anyTreeRank + : anyTreeRank, + }); + } else if (valueIsTreeDefinition(parentPartName)) + callbacks.handleTreeRanks({ + ...callbackPayload, + definitionName: getNameFromTreeDefinitionName(parentPartName), + }); else callbacks.handleSimpleFields(callbackPayload); const isSpecial = - valueIsToManyIndex(next.partName) || valueIsTreeRank(next.partName); + valueIsToManyIndex(next.partName) || valueIsTreeMeta(next.partName); + const nextField = isSpecial ? parentRelationship : table.getField(next.fieldName); @@ -141,6 +168,7 @@ function navigator({ if (typeof nextTable === 'object' && nextField?.isRelationship !== false) navigator({ + spec, callbacks, recursivePayload: { table: nextTable, @@ -285,7 +313,7 @@ export function getMappingLineData({ return { partName: nextPart, fieldName: - valueIsTreeRank(nextPart) || valueIsToManyIndex(nextPart) + valueIsTreeMeta(nextPart) || valueIsToManyIndex(nextPart) ? valueIsToManyIndex(mappingPath[internalState.position - 1]) ? mappingPath[internalState.position - 2] : mappingPath[internalState.position - 1] @@ -333,14 +361,65 @@ export function getMappingLineData({ ); }, - handleTreeRanks({ table }) { + handleTreeDefinitions({ table }) { + const defaultValue = getNameFromTreeDefinitionName( + internalState.defaultValue + ); + + commitInstanceData( + 'tree', + table, + generateFieldData === 'none' || + !hasTreeAccess(table.name as AnyTree['tableName'], 'read') + ? [] + : [ + spec.includeAnyTreeDefinition && + (generateFieldData === 'all' || + internalState.defaultValue === + formatTreeDefinition(anyTreeRank)) + ? [ + formatTreeDefinition(anyTreeRank), + { + optionLabel: queryText.anyTree(), + isRelationship: true, + isDefault: + internalState.defaultValue === + formatTreeDefinition(anyTreeRank), + isEnabled: true, + tableName: table.name, + }, + ] + : undefined, + ...(spec.includeSpecificTreeRanks + ? getTreeDefinitions( + table.name as AnyTree['tableName'], + 'all' + ).map(({ definition: { name } }) => + name === defaultValue || generateFieldData === 'all' + ? ([ + formatTreeDefinition(name), + { + optionLabel: name, + isRelationship: true, + isDefault: name === defaultValue, + tableName: table.name, + }, + ] as const) + : undefined + ) + : []), + ] + ); + }, + + handleTreeRanks({ table, definitionName }) { const defaultValue = getNameFromTreeRankName(internalState.defaultValue); commitInstanceData( 'tree', table, generateFieldData === 'none' || - !hasTreeAccess(table.name as 'Geography', 'read') + !hasTreeAccess(table.name as AnyTree['tableName'], 'read') ? [] : [ spec.includeAnyTreeRank && @@ -360,26 +439,28 @@ export function getMappingLineData({ ] : undefined, ...(spec.includeSpecificTreeRanks - ? getTreeDefinitions( - table.name as AnyTree['tableName'], - 'all' - ).flatMap(({ definition, ranks }) => - // Exclude the root rank for each tree - ranks.slice(1).map(({ name, title }) => - name === defaultValue || generateFieldData === 'all' - ? ([ - formatTreeRank(name), - { - optionLabel: title ?? name, - isRelationship: true, - isDefault: name === defaultValue, - tableName: table.name, - tableTreeDefName: definition.name, - }, - ] as const) - : undefined + ? getTreeDefinitions(table.name as AnyTree['tableName'], 'all') + .filter( + ({ definition: { name } }) => + definitionName === name || + definitionName === anyTreeRank + ) + .flatMap(({ ranks }) => + // Exclude the root rank for each tree + ranks.slice(1).map(({ name, title }) => + name === defaultValue || generateFieldData === 'all' + ? ([ + formatTreeRank(name), + { + optionLabel: title ?? name, + isRelationship: true, + isDefault: name === defaultValue, + tableName: table.name, + }, + ] as const) + : undefined + ) ) - ) : []), ] ); @@ -586,6 +667,7 @@ export function getMappingLineData({ }; navigator({ + spec, callbacks, baseTableName, }); @@ -595,7 +677,9 @@ export function getMappingLineData({ : internalState.mappingLineData.filter( ({ customSelectSubtype }) => customSelectSubtype !== 'toMany' ); - return spec.includeAnyTreeRank || spec.includeSpecificTreeRanks + return spec.includeAnyTreeRank || + spec.includeAnyTreeDefinition || + spec.includeSpecificTreeRanks ? filtered : filtered.filter( ({ customSelectSubtype }) => customSelectSubtype !== 'tree' diff --git a/specifyweb/frontend/js_src/lib/components/WbPlanView/navigatorSpecs.ts b/specifyweb/frontend/js_src/lib/components/WbPlanView/navigatorSpecs.ts index 643a974ab0d..3619bb9e853 100644 --- a/specifyweb/frontend/js_src/lib/components/WbPlanView/navigatorSpecs.ts +++ b/specifyweb/frontend/js_src/lib/components/WbPlanView/navigatorSpecs.ts @@ -14,6 +14,9 @@ export type NavigatorSpec = { // Whether no restrictions mode is enabled readonly isNoRestrictions: () => boolean; readonly includeToManyReferenceNumbers: boolean; + // REFACTOR: remove this attribute + readonly useSpecificTreeInterface: boolean; + readonly includeAnyTreeDefinition: boolean; readonly includeSpecificTreeRanks: boolean; readonly includeAnyTreeRank: boolean; readonly includeFormattedAggregated: boolean; @@ -40,7 +43,9 @@ const wbPlanView: NavigatorSpec = { isNoRestrictions: () => userPreferences.get('workBench', 'wbPlanView', 'noRestrictionsMode'), includeToManyReferenceNumbers: true, + useSpecificTreeInterface: true, includeSpecificTreeRanks: true, + includeAnyTreeDefinition: false, includeAnyTreeRank: false, includeFormattedAggregated: false, includeRootFormattedAggregated: false, @@ -81,6 +86,9 @@ const queryBuilder: NavigatorSpec = { isNoRestrictions: () => userPreferences.get('queryBuilder', 'general', 'noRestrictionsMode'), includeToManyReferenceNumbers: false, + // REFACTOR: see https://github.com/specify/specify7/pull/5251 + useSpecificTreeInterface: false, + includeAnyTreeDefinition: true, includeSpecificTreeRanks: true, includeAnyTreeRank: true, includeFormattedAggregated: true, @@ -111,6 +119,8 @@ const formatterEditor: NavigatorSpec = { includeReadOnly: true, isNoRestrictions: () => true, includeToManyReferenceNumbers: false, + useSpecificTreeInterface: false, + includeAnyTreeDefinition: false, includeSpecificTreeRanks: false, includeAnyTreeRank: false, includeFormattedAggregated: true, @@ -133,6 +143,8 @@ const permissive: NavigatorSpec = { includeReadOnly: true, isNoRestrictions: () => true, includeToManyReferenceNumbers: true, + useSpecificTreeInterface: true, + includeAnyTreeDefinition: true, includeSpecificTreeRanks: true, includeAnyTreeRank: true, includeFormattedAggregated: true, diff --git a/specifyweb/frontend/js_src/lib/components/WbPlanView/uploadPlanBuilder.ts b/specifyweb/frontend/js_src/lib/components/WbPlanView/uploadPlanBuilder.ts index 1d820221071..3082316a7e4 100644 --- a/specifyweb/frontend/js_src/lib/components/WbPlanView/uploadPlanBuilder.ts +++ b/specifyweb/frontend/js_src/lib/components/WbPlanView/uploadPlanBuilder.ts @@ -3,9 +3,14 @@ import { group, removeKey, split, toLowerCase } from '../../utils/utils'; import type { SpecifyTable } from '../DataModel/specifyTable'; import { strictGetTable } from '../DataModel/tables'; import type { Tables } from '../DataModel/types'; -import { isTreeTable } from '../InitialContext/treeRanks'; +import { getTreeDefinitions, isTreeTable } from '../InitialContext/treeRanks'; import { defaultColumnOptions } from './linesGetter'; import type { SplitMappingPath } from './mappingHelpers'; +import { + getNameFromTreeDefinitionName, + valueIsTreeDefinition, + valueIsTreeRank, +} from './mappingHelpers'; import { getNameFromTreeRankName, valueIsToManyIndex } from './mappingHelpers'; import type { ColumnDefinition, @@ -33,21 +38,64 @@ const toTreeRecordRanks = ( ): IR => Object.fromEntries( rankMappedFields.map(({ mappingPath, headerName, columnOptions }) => [ - mappingPath[0].toLowerCase(), + /* + * Use the last element of the mapping path to get the field name + * e.g: mappingPath when a tree is specified: ['$Kingdom', 'name'] vs when no tree is specified: ['name'] + */ + mappingPath.at(-1)?.toLowerCase(), toColumnOptions(headerName, columnOptions), ]) ); -const toTreeRecordVariety = (lines: RA): TreeRecord => ({ - ranks: Object.fromEntries( - indexMappings(lines).map(([fullRankName, rankMappedFields]) => [ - getNameFromTreeRankName(fullRankName), - { - treeNodeCols: toTreeRecordRanks(rankMappedFields), - }, - ]) - ), -}); +const toTreeRecordVariety = (lines: RA): TreeRecord => { + const treeDefinitions = getTreeDefinitions('Taxon', 'all'); + + return { + ranks: Object.fromEntries( + indexMappings(lines).flatMap(([fullName, rankMappedFields]) => { + /* + * For collections with only 1 tree, rankMappedFields already has the correct format for generating an upload plan + * When there are multiple trees in the mapper, rankMappedFields is actually a mapping path of trees mapped to ranks + * which means we need to get the index mappings of it again to get the actual rankMappedFields + */ + const rankMappings: RA]> = + valueIsTreeRank(fullName) + ? [[fullName, rankMappedFields]] + : indexMappings(rankMappedFields); + + const treeName: string | undefined = valueIsTreeDefinition(fullName) + ? getNameFromTreeDefinitionName(fullName) + : undefined; + + return rankMappings.map(([fullName, mappedFields]) => { + const rankName = getNameFromTreeRankName(fullName); + + // Resolve treeId using treeName or using rankName as fallback + const treeId = + treeName === undefined + ? treeDefinitions.find(({ ranks }) => + ranks.find((r) => r.name === rankName) + )?.definition.id + : treeDefinitions.find( + ({ definition }) => definition.name === treeName + )?.definition.id; + + return [ + treeId === undefined + ? rankName + : `${ + treeName === undefined ? '' : treeName + RANK_KEY_DELIMITER + }${rankName}`, + { + treeNodeCols: toTreeRecordRanks(mappedFields), + treeId, + }, + ]; + }); + }) + ), + }; +}; function toUploadTable( table: SpecifyTable, @@ -165,3 +213,6 @@ const indexMappings = ( ] as const ) ); + +// Delimiter used for rank name keys i.e: ~> +export const RANK_KEY_DELIMITER = '~>'; diff --git a/specifyweb/frontend/js_src/lib/components/WbPlanView/uploadPlanParser.ts b/specifyweb/frontend/js_src/lib/components/WbPlanView/uploadPlanParser.ts index 3a9463ca621..f5220b43d2e 100644 --- a/specifyweb/frontend/js_src/lib/components/WbPlanView/uploadPlanParser.ts +++ b/specifyweb/frontend/js_src/lib/components/WbPlanView/uploadPlanParser.ts @@ -3,10 +3,13 @@ import type { SpecifyTable } from '../DataModel/specifyTable'; import { strictGetTable } from '../DataModel/tables'; import type { Tables } from '../DataModel/types'; import { softFail } from '../Errors/Crash'; +import { getTreeDefinitions } from '../InitialContext/treeRanks'; import { defaultColumnOptions } from './linesGetter'; import type { MappingPath } from './Mapper'; import type { SplitMappingPath } from './mappingHelpers'; +import { formatTreeDefinition } from './mappingHelpers'; import { formatToManyIndex, formatTreeRank } from './mappingHelpers'; +import { RANK_KEY_DELIMITER } from './uploadPlanBuilder'; export type MatchBehaviors = 'ignoreAlways' | 'ignoreNever' | 'ignoreWhenBlank'; @@ -34,7 +37,10 @@ type UploadTableVariety = | { readonly uploadTable: UploadTable }; export type TreeRecord = { - readonly ranks: IR }>; + readonly ranks: IR< + | string + | { readonly treeNodeCols: IR; readonly treeId?: number } + >; }; type TreeRecordVariety = @@ -70,10 +76,23 @@ const parseTree = ( name: rankData, } : rankData.treeNodeCols, - [...mappingPath, formatTreeRank(rankName)] + [ + ...mappingPath, + ...(typeof rankData === 'object' && + typeof rankData.treeId === 'number' && + getTreeDefinitions('Taxon', 'all').length > 1 + ? [resolveTreeId(rankData.treeId)] + : []), + formatTreeRank(getRankNameWithoutTreeId(rankName)), + ] ) ); +const resolveTreeId = (id: number): string => { + const treeDefinition = getTreeDefinitions('Taxon', id); + return formatTreeDefinition(treeDefinition[0].definition.name); +}; + function parseTreeTypes( table: SpecifyTable, uploadPlan: TreeRecordVariety, @@ -194,3 +213,11 @@ export function parseUploadPlan(uploadPlan: UploadPlan): { ), }; } + +/** + * Returns the tree rank name after stripping its tree id: (e.x Kingdom~>1 => Kingdom) + * NOTE: Does not consider whether rankName is formatted with $ or not. + * Opposite of uploadPlanBuilder.ts > formatTreeRankWithTreeId() + */ +const getRankNameWithoutTreeId = (rankName: string): string => + rankName.split(RANK_KEY_DELIMITER)[0]; diff --git a/specifyweb/frontend/js_src/lib/localization/query.ts b/specifyweb/frontend/js_src/lib/localization/query.ts index 3e6785b534f..dc727eb393e 100644 --- a/specifyweb/frontend/js_src/lib/localization/query.ts +++ b/specifyweb/frontend/js_src/lib/localization/query.ts @@ -343,6 +343,9 @@ export const queryText = createDictionary({ 'uk-ua': '(будь-який ранг)', 'de-ch': '(jeder Rang)', }, + anyTree: { + 'en-us': '(any tree)', + }, moveUp: { comment: 'As in move it up', 'en-us': 'Move Up', diff --git a/specifyweb/frontend/js_src/lib/tests/fixtures/uploadplan.1.json b/specifyweb/frontend/js_src/lib/tests/fixtures/uploadplan.1.json index f0cc682997a..a639bc1327d 100644 --- a/specifyweb/frontend/js_src/lib/tests/fixtures/uploadplan.1.json +++ b/specifyweb/frontend/js_src/lib/tests/fixtures/uploadplan.1.json @@ -283,34 +283,40 @@ "default": null, "column": "Class" } - } + }, + "treeId": 1 }, "Family": { "treeNodeCols": { "name": "Family" - } + }, + "treeId": 1 }, "Genus": { "treeNodeCols": { "name": "Genus" - } + }, + "treeId": 1 }, "Subgenus": { "treeNodeCols": { "name": "Subgenus" - } + }, + "treeId": 1 }, "Species": { "treeNodeCols": { "name": "Species", "author": "Species Author" - } + }, + "treeId": 1 }, "Subspecies": { "treeNodeCols": { "name": "Subspecies", "author": "Subspecies Author" - } + }, + "treeId": 1 } } } diff --git a/specifyweb/specify/tree_utils.py b/specifyweb/specify/tree_utils.py index af442bd6787..ac9cd0783a0 100644 --- a/specifyweb/specify/tree_utils.py +++ b/specifyweb/specify/tree_utils.py @@ -5,6 +5,8 @@ lookup = lambda tree: (tree.lower() + 'treedef') +SPECIFY_TREES = {"taxon", "storage", "geography", "geologictimeperiod", "lithostrat"} + def get_search_filters(collection: spmodels.Collection, tree: str): tree_name = tree.lower() if tree_name == 'storage': diff --git a/specifyweb/workbench/tasks.py b/specifyweb/workbench/tasks.py index ea24552b954..295b75129bc 100644 --- a/specifyweb/workbench/tasks.py +++ b/specifyweb/workbench/tasks.py @@ -1,6 +1,7 @@ from typing import Optional, Any from celery import shared_task # type: ignore +from celery import Task # type: ignore from celery.utils.log import get_task_logger # type: ignore from django.db import connection, transaction @@ -14,37 +15,50 @@ logger = get_task_logger(__name__) -@app.task(base=LogErrorsTask, bind=True) -def upload(self, collection_id: int, uploading_agent_id: int, ds_id: int, no_commit: bool, allow_partial: bool) -> None: - - def progress(current: int, total: Optional[int]) -> None: - if not self.request.called_directly: - self.update_state(state='PROGRESS', meta={'current': current, 'total': total}) +def upload_data( + collection_id: int, + uploading_agent_id: int, + ds_id: int, + no_commit: bool, + allow_partial: bool, + task: Optional[Task]=None, + progress=None, +) -> None: with transaction.atomic(): ds = Spdataset.objects.select_for_update().get(id=ds_id) collection = Collection.objects.get(id=collection_id) - if ds.uploaderstatus is None: - logger.info("dataset is not assigned to an upload task") - return + if task is not None: + if ds.uploaderstatus is None: + logger.info("dataset is not assigned to an upload task") + return - if ds.uploaderstatus['taskid'] != self.request.id: - logger.info("dataset is not assigned to this task") - return + if ds.uploaderstatus['taskid'] != task.request.id: + logger.info("dataset is not assigned to this task") + return - if not (ds.uploaderstatus['operation'] == ("validating" if no_commit else "uploading")): raise AssertionError( - f"Invalid status '{ds.uploaderstatus['operation']}' for upload. Expected 'validating' or 'uploading'", - {"uploadStatus" : ds.uploaderstatus['operation'], - "operation" : "upload", - "expectedUploadStatus" : 'validating , uploading', - "localizationKey" : "invalidUploadStatus"}) + if not (ds.uploaderstatus['operation'] == ("validating" if no_commit else "uploading")): raise AssertionError( + f"Invalid status '{ds.uploaderstatus['operation']}' for upload. Expected 'validating' or 'uploading'", + {"uploadStatus" : ds.uploaderstatus['operation'], + "operation" : "upload", + "expectedUploadStatus" : 'validating , uploading', + "localizationKey" : "invalidUploadStatus"}) do_upload_dataset(collection, uploading_agent_id, ds, no_commit, allow_partial, progress) ds.uploaderstatus = None ds.save(update_fields=['uploaderstatus']) +@app.task(base=LogErrorsTask, bind=True) +def upload(self, collection_id: int, uploading_agent_id: int, ds_id: int, no_commit: bool, allow_partial: bool) -> None: + + def progress(current: int, total: Optional[int]) -> None: + if not self.request.called_directly: + self.update_state(state='PROGRESS', meta={'current': current, 'total': total}) + + upload_data(collection_id, uploading_agent_id, ds_id, no_commit, allow_partial, self, progress) + @app.task(base=LogErrorsTask, bind=True) def unupload(self, ds_id: int, agent_id: int) -> None: diff --git a/specifyweb/workbench/upload/scoping.py b/specifyweb/workbench/upload/scoping.py index 2d84a98a8ba..c75d90d4b16 100644 --- a/specifyweb/workbench/upload/scoping.py +++ b/specifyweb/workbench/upload/scoping.py @@ -9,7 +9,7 @@ from .uploadable import Uploadable, ScopedUploadable from .upload_table import UploadTable, DeferredScopeUploadTable, ScopedUploadTable, OneToOneTable, ScopedOneToOneTable from .tomany import ToManyRecord, ScopedToManyRecord -from .treerecord import TreeRecord, ScopedTreeRecord +from .treerecord import TreeRank, TreeRankRecord, TreeRecord, ScopedTreeRecord from .column_options import ColumnOptions, ExtendedColumnOptions """ There are cases in which the scoping of records should be dependent on another record/column in a WorkBench dataset. @@ -186,10 +186,10 @@ def apply_scoping_to_tomanyrecord(tmr: ToManyRecord, collection) -> ScopedToMany def apply_scoping_to_treerecord(tr: TreeRecord, collection) -> ScopedTreeRecord: table = datamodel.get_table_strict(tr.name) + treedef = None if table.name == 'Taxon': - # TODO: Add scoping in the workbench via the Upload Plan - # treedef = get_taxon_treedef(collection) - treedef = collection.discipline.taxontreedef + if treedef is None: + treedef = collection.discipline.taxontreedef elif table.name == 'Geography': treedef = collection.discipline.geographytreedef @@ -206,19 +206,36 @@ def apply_scoping_to_treerecord(tr: TreeRecord, collection) -> ScopedTreeRecord: else: raise Exception(f'unexpected tree type: {table.name}') + if treedef is None: + raise ValueError(f"Could not find treedef for table {table.name}") + treedefitems = list(treedef.treedefitems.order_by('rankid')) treedef_ranks = [tdi.name for tdi in treedefitems] for rank in tr.ranks: - if rank not in treedef_ranks: + if rank not in treedef_ranks and isinstance(rank, TreeRankRecord) and not rank.check_rank(table.name): raise Exception(f'"{rank}" not among {table.name} tree ranks: {treedef_ranks}') root = list(getattr(models, table.name.capitalize()).objects.filter(definitionitem=treedefitems[0])[:1]) # assume there is only one + scoped_ranks: Dict[TreeRankRecord, Dict[str, ExtendedColumnOptions]] = { + ( + TreeRank.create( + r, table.name, treedef.id if treedef else None + ).tree_rank_record() + if isinstance(r, str) + else r + ): { + f: extend_columnoptions(colopts, collection, table.name, f) + for f, colopts in cols.items() + } + for r, cols in tr.ranks.items() + } + return ScopedTreeRecord( name=tr.name, - ranks={r: {f: extend_columnoptions(colopts, collection, table.name, f) for f, colopts in cols.items()} for r, cols in tr.ranks.items()}, + ranks=scoped_ranks, treedef=treedef, - treedefitems=list(treedef.treedefitems.order_by('rankid')), + treedefitems=treedefitems, root=root[0] if root else None, disambiguation={}, ) diff --git a/specifyweb/workbench/upload/tests/testschema.py b/specifyweb/workbench/upload/tests/testschema.py index 3b9d1117e68..111776a8e93 100644 --- a/specifyweb/workbench/upload/tests/testschema.py +++ b/specifyweb/workbench/upload/tests/testschema.py @@ -14,10 +14,19 @@ from .base import UploadTestsBase from . import example_plan +def set_plan_treeId(): + # Set the treeId in example_plan.json dynamically, so that the unit test doesn't depend on the static treeId to always be the same. + from specifyweb.specify.models import Taxontreedefitem + tree_id = Taxontreedefitem.objects.filter(name='Species').first().treedef_id + example_plan_ranks = example_plan.json['uploadable']['uploadTable']['toMany']['determinations'][0]['toOne']['taxon']['treeRecord']['ranks'] + for rank in ['Species', 'Subspecies']: + example_plan_ranks[rank]['treeId'] = tree_id + class SchemaTests(UploadTestsBase): maxDiff = None def test_schema_parsing(self) -> None: + set_plan_treeId() Draft7Validator.check_schema(schema) validate(example_plan.json, schema) plan = parse_plan(self.collection, example_plan.json).apply_scoping(self.collection) @@ -26,6 +35,7 @@ def test_schema_parsing(self) -> None: self.assertEqual(repr(plan), repr(self.example_plan)) def test_unparsing(self) -> None: + set_plan_treeId() self.assertEqual(example_plan.json, parse_plan(self.collection, example_plan.json).unparse()) def test_reject_internal_tree_columns(self) -> None: diff --git a/specifyweb/workbench/upload/treerecord.py b/specifyweb/workbench/upload/treerecord.py index d34f7619aa5..92e29df7729 100644 --- a/specifyweb/workbench/upload/treerecord.py +++ b/specifyweb/workbench/upload/treerecord.py @@ -2,7 +2,10 @@ For uploading tree records. """ +from itertools import groupby import logging +from math import e +from os import error from typing import List, Dict, Any, Tuple, NamedTuple, Optional, Union, Set from django.db import transaction, IntegrityError @@ -10,6 +13,7 @@ from specifyweb.businessrules.exceptions import BusinessRuleException from specifyweb.specify import models +from specifyweb.specify.tree_utils import SPECIFY_TREES from .column_options import ColumnOptions, ExtendedColumnOptions from .parsing import ParseResult, WorkBenchParseFailure, parse_many, filter_and_upload from .upload_result import UploadResult, NullRecord, NoMatch, Matched, \ @@ -19,33 +23,241 @@ logger = logging.getLogger(__name__) +class TreeRankCell(NamedTuple): + treedef_id: int + treedefitem_name: str + tree_node_attribute: str + upload_value: str + column_fullname: str = "" + +class TreeRank(NamedTuple): + rank_name: str + treedef_id: int + tree: str + + @staticmethod + def create( + rank_name: str, + tree: str, + treedef_id: Optional[int] = None, + ) -> 'TreeRank': + """ + Create a TreeRank instance with the given rank name, tree, and optional treedef IDs. + """ + tree_model = get_treedefitem_model(tree) + + def build_filter_kwargs(rank_name: str, treedef_id: Optional[int] = None) -> Dict[str, Any]: + """ + Build filter keyword arguments for querying treedef items. + """ + + # Create base keyword arguments for filtering + def create_base_kwargs(rank_name: str) -> Dict[str, Any]: + return {'name': rank_name} + + # Add treedef ID to keyword arguments if present + def add_treedef_id_if_present(kwargs: Dict[str, Any], treedef_id: Optional[int]) -> Dict[str, Any]: + if treedef_id is not None: + kwargs['treedef_id'] = treedef_id + return kwargs + + filter_kwargs = create_base_kwargs(rank_name) + filter_kwargs = add_treedef_id_if_present(filter_kwargs, treedef_id) + return filter_kwargs + + def filter_by_treedef_id(treedefitems, rank_name: str): + """ + Filter treedefitems by treedef_id and ensure only one item is found. + """ + + # Check if treedef ID is present + def check_treedef_id(treedef_id): + if treedef_id is None: + raise ValueError(f"Multiple treedefitems found for rank {rank_name}") + + # Filter treedefitems by treedef ID + def filter_items_by_treedef_id(treedefitems, treedef_id): + return treedefitems.filter(treedef_id=treedef_id) + + # Validate filtered items + def validate_filtered_items(treedefitems): + if not treedefitems.exists(): + raise ValueError( + f"No treedefitems found for rank {rank_name} and treedef {treedef_id}" + ) + if treedefitems.count() > 1: + raise ValueError( + f"Multiple treedefitems found for rank {rank_name} and treedef {treedef_id}" + ) + + # Check base treedef ID and filter treedefitems, then validate + check_treedef_id(treedef_id) + filtered_items = filter_items_by_treedef_id(treedefitems, treedef_id) + validate_filtered_items(filtered_items) + return filtered_items + + def get_treedef_id( + rank_name: str, + tree: str, + treedef_name: Optional[str] = None, + treedef_id: Optional[int] = None + ) -> int: + """ + Get the treedef ID for the given rank name and tree. + """ + + # Fetch treedefitems based on filter keyword arguments + def fetch_treedefitems(filter_kwargs): + return tree_model.objects.filter(**filter_kwargs) + + # Get the first item from the queryset + def get_first_item(treedefitems): + first_item = treedefitems.first() + if first_item is None: + raise ValueError(f"No treedefitems found for rank {rank_name}") + return first_item + + # Handle cases where multiple items are found + def handle_multiple_items(treedefitems): + if treedefitems.count() > 1: + return filter_by_treedef_id(treedefitems, rank_name, treedef_id) + return treedefitems + + if treedef_id is None and treedef_name is not None: + treedef_id = get_treedef_model(tree).objects.get(name=treedef_name) + + # Build filter keyword arguments and fetch treedefitems + filter_kwargs = build_filter_kwargs(rank_name, treedef_id) + treedefitems = fetch_treedefitems(filter_kwargs) + + # Handle cases where no items are found + if not treedefitems.exists() and treedef_id is not None: + filter_kwargs = build_filter_kwargs(rank_name) + treedefitems = fetch_treedefitems(filter_kwargs) + + treedefitems = handle_multiple_items(treedefitems) + first_item = get_first_item(treedefitems) + + return first_item.treedef_id + + def extract_treedef_name(rank_name: str) -> Tuple[str, Optional[str]]: + """ + Extract treedef_name from rank_name if it exists in the format 'treedef_name~>rank_name'. + """ + parts = rank_name.split('~>', 1) + if len(parts) == 2: + treedef_name = parts[0] + rank_name = parts[1] + return rank_name, treedef_name + return rank_name, None + + rank_name, extracted_treedef_name = extract_treedef_name(rank_name) + target_treedef_id = get_treedef_id(rank_name, tree, extracted_treedef_name, treedef_id) + + return TreeRank(rank_name, target_treedef_id, tree.lower()) + + def check_rank(self) -> bool: + """ + Check if the rank exists and is unique for the given tree and treedef ID. + """ + + # Get the tree model + def get_tree_model(tree: str): + return get_treedefitem_model(tree) + + # Filter the rank by name + def filter_rank(tree_model, rank_name: str, treedef_id: int): + return tree_model.objects.filter(name=rank_name, treedef_id=treedef_id) + + # Check if the rank exists and is unique + def rank_exists_and_unique(rank): + return rank.exists() and rank.count() == 1 + + tree_model = get_tree_model(self.tree) + rank = filter_rank(tree_model, self.rank_name, self.treedef_id) + return rank_exists_and_unique(rank) + + def validate_rank(self) -> None: + """ + Validate the rank for the given tree and treedef ID. + """ + + if not self.check_rank(): + raise ValueError(f"Invalid rank {self.rank_name} for treedef {self.treedef_id}") + + def tree_rank_record(self) -> 'TreeRankRecord': + """ + Create a TreeRankRecord instance. + """ + + assert self.rank_name is not None, "Rank name is required" + assert self.treedef_id is not None, "Treedef ID is required" + assert self.tree is not None and self.tree.lower() in SPECIFY_TREES, "Tree is required" + return TreeRankRecord(self.rank_name, self.treedef_id) + +def get_treedefitem_model(tree: str): + return getattr(models, tree.lower().title() + 'treedefitem') + +def get_treedef_model(tree: str): + return getattr(models, tree.lower().title() + 'treedef') + +class TreeRankRecord(NamedTuple): + rank_name: str + treedef_id: int + + # Create a TreeRankRecord instance + def to_json(self) -> Dict: + return { + "rank": self.rank_name, + "treedefId": self.treedef_id, + } + + # Get the key for the TreeRankRecord instance + def to_key(self) -> Tuple[str, int]: + return (self.rank_name, self.treedef_id) + + # Check if the rank exists and is unique for the given tree and treedef ID + def check_rank(self, tree: str) -> bool: + return TreeRank.create(self.rank_name, tree, self.treedef_id).check_rank() + + # Validate the rank for the given tree and treedef ID + def validate_rank(self, tree) -> None: + TreeRank.create(self.rank_name, tree, self.treedef_id).validate_rank() class TreeRecord(NamedTuple): name: str - ranks: Dict[str, Dict[str, ColumnOptions]] + ranks: Dict[Union[str, TreeRankRecord], Dict[str, ColumnOptions]] def apply_scoping(self, collection) -> "ScopedTreeRecord": from .scoping import apply_scoping_to_treerecord as apply_scoping return apply_scoping(self, collection) def get_cols(self) -> Set[str]: - return set(col.column for r in self.ranks.values() for col in r.values()) + return {col.column for r in self.ranks.values() for col in r.values() if hasattr(col, 'column')} def to_json(self) -> Dict: - result = { - 'ranks': { - rank: cols['name'].to_json() if len(cols) == 1 else dict(treeNodeCols={k: v.to_json() for k, v in cols.items()}) - for rank, cols in self.ranks.items() - }, - } - return { 'treeRecord': result } + result = {"ranks": {}} # type: ignore + + for rank, cols in self.ranks.items(): + rank_key = rank.rank_name if isinstance(rank, TreeRankRecord) else rank + treeNodeCols = {k: v.to_json() if hasattr(v, "to_json") else v for k, v in cols.items()} + + if len(cols) == 1: + result["ranks"][rank_key] = treeNodeCols["name"] + else: + rank_data = {"treeNodeCols": treeNodeCols} + if isinstance(rank, TreeRankRecord): + rank_data["treeId"] = rank.treedef_id # type: ignore + result["ranks"][rank_key] = rank_data + + return {'treeRecord': result} def unparse(self) -> Dict: return { 'baseTableName': self.name, 'uploadble': self.to_json() } class ScopedTreeRecord(NamedTuple): name: str - ranks: Dict[str, Dict[str, ExtendedColumnOptions]] + ranks: Dict[TreeRankRecord, Dict[str, ExtendedColumnOptions]] treedef: Any treedefitems: List root: Optional[Any] @@ -55,20 +267,189 @@ def disambiguate(self, disambiguation: DA) -> "ScopedTreeRecord": return self._replace(disambiguation=disambiguation.disambiguate_tree()) if disambiguation is not None else self def get_treedefs(self) -> Set: - return set([self.treedef]) + # return set([self.treedef]) # old way + tree_def_model = get_treedef_model(self.name) + treedefids = set([tree_rank_record.treedef_id for tree_rank_record in self.ranks.keys()]) + return set(tree_def_model.objects.filter(id__in=treedefids)) + + def _get_not_null_ranks_columns_in_row(self, row: Row) -> List[TreeRankCell]: + """ + Get rank columns that are not null in the row. + """ + + # Check if the value is not null + def is_not_null(value: Any) -> bool: + return bool(value) + + # Format the column name + def format_column(col_name: str, col_opts: Any) -> str: + return f'{col_opts.column} - {col_name}' if col_name != 'name' else col_opts.column + + # Match the column + def match_column(row_key: str, formatted_column: str) -> bool: + return formatted_column == row_key + + # Create a TreeRankCell instance + def create_tree_rank_cell(tree_rank_record: Any, col_name: str, row_value: Any, row_key: Any) -> TreeRankCell: + return TreeRankCell( + tree_rank_record.treedef_id, # treedefid + tree_rank_record.rank_name, # treedefitem_name + col_name, # tree_node_attribute + row_value, # upload_value + row_key # column_fullname + ) + + # Process the row item + def process_row_item(row_key: str, row_value: Any) -> List[TreeRankCell]: + if not is_not_null(row_value): + return [] + return [ + create_tree_rank_cell(tree_rank_record, col_name, row_value, row_key) + for tree_rank_record, cols in self.ranks.items() + for col_name, col_opts in cols.items() + if match_column(row_key, format_column(col_name, col_opts)) + ] - def bind(self, collection, row: Row, uploadingAgentId: Optional[int], auditor: Auditor, cache: Optional[Dict]=None, row_index: Optional[int] = None) -> Union["BoundTreeRecord", ParseFailures]: - parsedFields: Dict[str, List[ParseResult]] = {} + return [ + cell + for row_key, row_value in row.items() + for cell in process_row_item(row_key, row_value) + ] + + def rescope_tree_from_row(self, row: Row) -> Tuple["ScopedTreeRecord", Optional["WorkBenchParseFailure"]]: + """Rescope tree from row data.""" + + # Get models based on the name + def get_models(name: str): + tree_def_model = get_treedef_model(name) + tree_rank_model = get_treedefitem_model(name) + tree_node_model = getattr(models, name.lower().title()) + return tree_def_model, tree_rank_model, tree_node_model + + # Get rank columns that are not null in the row + def get_not_null_ranks_columns(row: Row) -> List[TreeRankCell]: + return self._get_not_null_ranks_columns_in_row(row) + + # Determine the target treedef based on the columns that are not null + def get_targeted_treedefids(ranks_columns: List[TreeRankCell]) -> Set[int]: + return set(rank_column.treedef_id for rank_column in ranks_columns) + + # Handle cases where there are multiple or no treedefs + def handle_multiple_or_no_treedefs( + targeted_treedefids: Set[int], ranks_columns: List[TreeRankCell] + ) -> Optional[Tuple["ScopedTreeRecord", Optional["WorkBenchParseFailure"]]]: + if not targeted_treedefids: + return self, None + elif len(targeted_treedefids) > 1: + logger.warning(f"Multiple treedefs found in row: {targeted_treedefids}") + error_col_name = ranks_columns[0].column_fullname + return self, WorkBenchParseFailure('multipleRanksInRow', {}, error_col_name) + + # Group ranks_columns by treedef_id using itertools.groupby + ranks_columns.sort(key=lambda x: x.treedef_id) # groupby requires sorted input + grouped_by_treedef_id = {k: list(v) for k, v in groupby(ranks_columns, key=lambda x: x.treedef_id)} + + # Check if any treedef_id has more than one rank + multiple_ranks = any(len(columns) > 1 for columns in grouped_by_treedef_id.values()) + if multiple_ranks: + treedef_id = next( + treedef_id + for treedef_id, columns in grouped_by_treedef_id.items() + if len(columns) > 1 + ) + logger.warning(f"Multiple ranks found for treedef_id {treedef_id}") + error_col_name = grouped_by_treedef_id[treedef_id][0].column_fullname + return self, WorkBenchParseFailure("multipleRanksForTreedef", {}, error_col_name) + + return None + + # Retrieve the target rank treedef + def get_target_rank_treedef(tree_def_model, target_rank_treedef_id: int): + return tree_def_model.objects.get(id=target_rank_treedef_id) + + # Retrieve the treedef items and root for the tree + def get_treedefitems_and_root(tree_rank_model, tree_node_model, target_rank_treedef_id: int): + # Fetch treedef items + def fetch_treedefitems(): + return list(tree_rank_model.objects.filter(treedef_id=target_rank_treedef_id).order_by("rankid")) + + # Fetch root node + def fetch_root(): + return tree_node_model.objects.filter(definition_id=target_rank_treedef_id, parent=None).first() + + # Check if root is None and log warning + def check_root(root): + if root is None: + logger.warning(f"No root found for treedef {target_rank_treedef_id}") + return None, WorkBenchParseFailure('noRoot', {}, None) + return root + + treedefitems = fetch_treedefitems() + root = fetch_root() + root_checked = check_root(root) + + if root_checked is not root: + return root_checked + + return treedefitems, root + + tree_def_model, tree_rank_model, tree_node_model = get_models(self.name) + + unique_treedef_ids = {tr.treedef_id for tr in self.ranks.keys()} + if len(unique_treedef_ids) == 1: + return self, None + + ranks_columns_in_row_not_null = get_not_null_ranks_columns(row) + targeted_treedefids = get_targeted_treedefids(ranks_columns_in_row_not_null) + + result = handle_multiple_or_no_treedefs(targeted_treedefids, ranks_columns_in_row_not_null) + if result: + return result + + # Determine the target treedef based on the columns that are not null + targeted_treedefids = {rank_column.treedef_id for rank_column in ranks_columns_in_row_not_null} + if not targeted_treedefids: + # return self, WorkBenchParseFailure('noRanksInRow', {}, None) + return self, None + elif len(targeted_treedefids) > 1: + logger.warning(f"Multiple treedefs found in row: {targeted_treedefids}") + error_col_name = list(ranks_columns_in_row_not_null)[0].column_fullname + return self, WorkBenchParseFailure("multipleRanksInRow", {}, error_col_name) + + target_rank_treedef_id = targeted_treedefids.pop() + target_rank_treedef = get_target_rank_treedef(tree_def_model, target_rank_treedef_id) + + # Based on the target treedef, get the treedefitems and root for the tree + treedefitems = list(tree_rank_model.objects.filter(treedef_id=target_rank_treedef_id).order_by("rankid")) + root = tree_node_model.objects.filter(definition_id=target_rank_treedef_id, parent=None).first() + + return self._replace(treedef=target_rank_treedef, treedefitems=treedefitems, root=root), None + + def bind( + self, + collection, + row: Row, + uploadingAgentId: Optional[int], + auditor: Auditor, + cache: Optional[Dict] = None, + row_index: Optional[int] = None, + ) -> Union["BoundTreeRecord", ParseFailures]: + parsedFields: Dict[TreeRankRecord, List[ParseResult]] = {} parseFails: List[WorkBenchParseFailure] = [] - for rank, cols in self.ranks.items(): + + rescoped_tree_record, parse_fail = self.rescope_tree_from_row(row) + if parse_fail: + parseFails.append(parse_fail) + + for tree_rank_record, cols in self.ranks.items(): nameColumn = cols['name'] presults, pfails = parse_many(collection, self.name, cols, row) - parsedFields[rank] = presults + parsedFields[tree_rank_record] = presults parseFails += pfails filters = {k: v for result in presults for k, v in result.filter_on.items()} if filters.get('name', None) is None: parseFails += [ - WorkBenchParseFailure('invalidPartialRecord',{'column':nameColumn.column}, result.column) + WorkBenchParseFailure('invalidPartialRecord', {'column': nameColumn.column}, result.column) for result in presults if any(v is not None for v in result.filter_on.values()) ] @@ -78,9 +459,9 @@ def bind(self, collection, row: Row, uploadingAgentId: Optional[int], auditor: A return BoundTreeRecord( name=self.name, - treedef=self.treedef, - treedefitems=self.treedefitems, - root=self.root, + treedef=rescoped_tree_record.treedef, + treedefitems=rescoped_tree_record.treedefitems, + root=rescoped_tree_record.root, disambiguation=self.disambiguation, parsedFields=parsedFields, uploadingAgentId=uploadingAgentId, @@ -88,6 +469,7 @@ def bind(self, collection, row: Row, uploadingAgentId: Optional[int], auditor: A cache=cache, ) + class MustMatchTreeRecord(TreeRecord): def apply_scoping(self, collection) -> "ScopedMustMatchTreeRecord": s = super().apply_scoping(collection) @@ -114,7 +496,7 @@ class BoundTreeRecord(NamedTuple): treedef: Any treedefitems: List root: Optional[Any] - parsedFields: Dict[str, List[ParseResult]] + parsedFields: Dict[TreeRankRecord, List[ParseResult]] uploadingAgentId: Optional[int] auditor: Auditor cache: Optional[Dict] @@ -157,10 +539,30 @@ def _handle_row(self, must_match: bool) -> UploadResult: return UploadResult(match_result, {}, {}) def _to_match(self) -> List[TreeDefItemWithParseResults]: + + # Check if the parse results have non-null values + def has_non_null_values(parse_results): + return any( + value is not None + for result in parse_results + for value in result.filter_on.values() + ) + + # Get parse results for a TDI + def get_parse_results(tdi): + tree_rank_record = TreeRankRecord(tdi.name, tdi.treedef_id) + return self.parsedFields.get(tree_rank_record, []) + + # Check if the TDI is valid + def is_valid_tdi(tdi): + tree_rank_names = {trr.rank_name for trr in self.parsedFields.keys()} + return tdi.name in tree_rank_names and has_non_null_values(get_parse_results(tdi)) + + # Create a TreeDefItemWithParseResults instance return [ - TreeDefItemWithParseResults(tdi, self.parsedFields[tdi.name]) + TreeDefItemWithParseResults(tdi, get_parse_results(tdi)) for tdi in self.treedefitems - if tdi.name in self.parsedFields and any(v is not None for r in self.parsedFields[tdi.name] for v in r.filter_on.values()) + if is_valid_tdi(tdi) ] def _match(self, tdiwprs: List[TreeDefItemWithParseResults]) -> Tuple[List[TreeDefItemWithParseResults], MatchResult]: diff --git a/specifyweb/workbench/upload/upload_plan_schema.py b/specifyweb/workbench/upload/upload_plan_schema.py index 3826fbf4263..e148307fa46 100644 --- a/specifyweb/workbench/upload/upload_plan_schema.py +++ b/specifyweb/workbench/upload/upload_plan_schema.py @@ -1,16 +1,18 @@ -from typing import Dict, Any, Union, Tuple +from functools import reduce +from os import name +from typing import Dict, Any, Optional, Union, Tuple +import logging from specifyweb.specify.datamodel import datamodel, Table, Relationship -from specifyweb.specify.load_datamodel import DoesNotExistError -from specifyweb.specify import models from .upload_table import DeferredScopeUploadTable, UploadTable, OneToOneTable, MustMatchTable from .tomany import ToManyRecord -from .treerecord import TreeRecord, MustMatchTreeRecord +from .treerecord import TreeRank, TreeRankRecord, TreeRecord, MustMatchTreeRecord from .uploadable import Uploadable from .column_options import ColumnOptions from .scoping import DEFERRED_SCOPING +logger = logging.getLogger(__name__) schema: Dict = { 'title': 'Specify 7 Workbench Upload Plan', @@ -33,7 +35,7 @@ 'additionalProperties': False }, ] - }, + } }, 'required': [ 'baseTableName', 'uploadable' ], 'additionalProperties': False, @@ -91,6 +93,7 @@ 'type': 'object', 'properties': { 'treeNodeCols': { '$ref': '#/definitions/treeNodeCols' }, + 'treeId': { '$ref': '#definitions/treeId'} }, 'required': [ 'treeNodeCols' ], 'additionalProperties': False @@ -140,6 +143,13 @@ ], }, + 'treeId': { + "oneOf": [ + { "type": "integer" }, + { "type": "null" } + ] + }, + 'wbcols': { 'type': 'object', 'description': 'Maps the columns of the destination table to the headers of the source columns of input data.', @@ -288,17 +298,45 @@ def defer_scope_upload_table(default_collection, table: Table, to_parse: Dict, d ) def parse_tree_record(collection, table: Table, to_parse: Dict) -> TreeRecord: - ranks = { - rank: {'name': parse_column_options(name_or_cols)} if isinstance(name_or_cols, str) - else {k: parse_column_options(v) for k,v in name_or_cols['treeNodeCols'].items() } - for rank, name_or_cols in to_parse['ranks'].items() - } - for rank, cols in ranks.items(): - assert 'name' in cols, to_parse + """Parse tree record from the given data""" + + def parse_rank(name_or_cols): + if isinstance(name_or_cols, str): + return name_or_cols, {'name': parse_column_options(name_or_cols)}, None + tree_node_cols = {k: parse_column_options(v) for k, v in name_or_cols['treeNodeCols'].items()} + rank_name = name_or_cols['treeNodeCols']['name'] + treedefid = name_or_cols.get('treeId') + return rank_name, tree_node_cols, treedefid + + def create_tree_rank_record(rank, table_name, treedefid): + return TreeRank.create(rank, table_name, treedefid).tree_rank_record() + + def parse_ranks(to_parse, table_name): + def parse_single_rank(rank, name_or_cols): + rank_name, parsed_cols, treedefid = parse_rank(name_or_cols) + tree_rank_record = create_tree_rank_record(rank, table_name, treedefid) + return tree_rank_record, parsed_cols + + def aggregate_ranks(acc, item): + rank, name_or_cols = item + tree_rank_record, parsed_cols = parse_single_rank(rank, name_or_cols) + acc[tree_rank_record] = parsed_cols + return acc + + ranks = reduce(aggregate_ranks, to_parse['ranks'].items(), {}) + return ranks + + # Validate ranks by checking if 'name' is present in each rank + def validate_ranks(ranks, to_parse): + for rank, cols in ranks.items(): + assert 'name' in cols, to_parse + + ranks = parse_ranks(to_parse, table.name) + validate_ranks(ranks, to_parse) return TreeRecord( name=table.django_name, - ranks=ranks, + ranks=ranks ) def parse_to_many_record(collection, table: Table, to_parse: Dict) -> ToManyRecord: diff --git a/specifyweb/workbench/views.py b/specifyweb/workbench/views.py index b77dc83a1ee..a1f94d3e2d0 100644 --- a/specifyweb/workbench/views.py +++ b/specifyweb/workbench/views.py @@ -480,8 +480,9 @@ def dataset(request, ds: models.Spdataset) -> http.HttpResponse: # TODO fix this # return http.HttpResponse(f"upload plan is invalid: {e}", status=400) pass - - new_cols = upload_plan_schema.parse_plan(request.specify_collection, plan).get_cols() - set(ds.columns) + + parsed_plan = upload_plan_schema.parse_plan(request.specify_collection, plan) + new_cols = parsed_plan.get_cols() - set(ds.columns) if new_cols: ncols = len(ds.columns) ds.columns += list(new_cols) @@ -626,22 +627,35 @@ def upload(request, ds, no_commit: bool, allow_partial: bool) -> http.HttpRespon if ds.was_uploaded(): return http.HttpResponse('dataset has already been uploaded.', status=400) - taskid = str(uuid4()) - async_result = tasks.upload.apply_async([ - request.specify_collection.id, - request.specify_user_agent.id, - ds.id, - no_commit, - allow_partial - ], task_id=taskid) - ds.uploaderstatus = { - 'operation': "validating" if no_commit else "uploading", - 'taskid': taskid - } - ds.save(update_fields=['uploaderstatus']) - - return http.JsonResponse(async_result.id, safe=False) - + data = json.loads(request.body) if request.body else {} + background = True + if 'background' in data: + background = bool(data['background']) # {"background": false} + + if background: + taskid = str(uuid4()) + async_result = tasks.upload.apply_async([ + request.specify_collection.id, + request.specify_user_agent.id, + ds.id, + no_commit, + allow_partial + ], task_id=taskid) + ds.uploaderstatus = { + 'operation': "validating" if no_commit else "uploading", + 'taskid': taskid + } + ds.save(update_fields=['uploaderstatus']) + return http.JsonResponse(async_result.id, safe=False) + else: + tasks.upload_data( + request.specify_collection.id, + request.specify_user_agent.id, + ds.id, + no_commit, + allow_partial, + ) + return http.JsonResponse(None, safe=False) @openapi(schema={ 'post': {