From 3971c3169a689ccc0b2879d82408cc06ba4301f9 Mon Sep 17 00:00:00 2001 From: Caroline D <108160931+CarolineDenis@users.noreply.github.com> Date: Wed, 10 Jul 2024 13:22:27 -0700 Subject: [PATCH 01/78] Add a feature to choose type of taxon tree in WB Fixes #4980 --- .../lib/components/WbPlanView/Wrapped.tsx | 66 +++++++++++++++---- .../js_src/lib/localization/workbench.ts | 3 + 2 files changed, 57 insertions(+), 12 deletions(-) diff --git a/specifyweb/frontend/js_src/lib/components/WbPlanView/Wrapped.tsx b/specifyweb/frontend/js_src/lib/components/WbPlanView/Wrapped.tsx index 4f6e50edefa..8ec8c97747b 100644 --- a/specifyweb/frontend/js_src/lib/components/WbPlanView/Wrapped.tsx +++ b/specifyweb/frontend/js_src/lib/components/WbPlanView/Wrapped.tsx @@ -9,11 +9,26 @@ import { useNavigate } from 'react-router-dom'; import type { LocalizedString } from 'typesafe-i18n'; import type { State } from 'typesafe-reducer'; +import { usePromise } from '../../hooks/useAsyncState'; +import { useCachedState } from '../../hooks/useCachedState'; import { useErrorContext } from '../../hooks/useErrorContext'; import { useLiveState } from '../../hooks/useLiveState'; +import { commonText } from '../../localization/common'; +import { resourcesText } from '../../localization/resources'; +import { treeText } from '../../localization/tree'; +import { wbText } from '../../localization/workbench'; +import { f } from '../../utils/functools'; import type { IR, RA } from '../../utils/types'; +import { caseInsensitiveHash } from '../../utils/utils'; +import { Button } from '../Atoms/Button'; +import { Select } from '../Atoms/Form'; import type { Tables } from '../DataModel/types'; +import { + getTreeDefinitions, + treeRanksPromise, +} from '../InitialContext/treeRanks'; import { useTitle } from '../Molecules/AppTitle'; +import { Dialog } from '../Molecules/Dialog'; import { ProtectedAction } from '../Permissions/PermissionDenied'; import type { UploadResult } from '../WorkBench/resultsParser'; import { savePlan } from './helpers'; @@ -121,25 +136,37 @@ export function WbPlanView({ ); useErrorContext('state', state); + const [isTaxonTable, setIsTaxonTable] = React.useState(false); + + const [treeDefinitions] = usePromise(treeRanksPromise, true); + const definitionsForTree = treeDefinitions + ? caseInsensitiveHash(treeDefinitions, 'Taxon') + : undefined; + const definitions = definitionsForTree?.map(({ definition }) => definition); + const navigate = useNavigate(); return state.type === 'SelectBaseTable' ? ( navigate(`/specify/workbench/${dataset.id}/`)} - onSelected={(baseTableName): void => - setState({ - type: 'MappingState', - changesMade: true, - baseTableName, - lines: getLinesFromHeaders({ - headers, - runAutoMapper: true, + onSelected={(baseTableName): void => { + if (baseTableName === 'Taxon') { + setIsTaxonTable(true); + } else { + setState({ + type: 'MappingState', + changesMade: true, baseTableName, - }), - mustMatchPreferences: {}, - }) - } + lines: getLinesFromHeaders({ + headers, + runAutoMapper: true, + baseTableName, + }), + mustMatchPreferences: {}, + }); + } + }} onSelectTemplate={(uploadPlan, headers): void => setState({ type: 'MappingState', @@ -148,6 +175,21 @@ export function WbPlanView({ }) } /> + {isTaxonTable && ( + {commonText.close()} + } + header={wbText.selectTree()} + onClose={(): void => setIsTaxonTable(false)} + > + {definitions?.map(({ id, name }) => ( + + ))} + + )} ) : ( Date: Wed, 10 Jul 2024 14:04:17 -0700 Subject: [PATCH 02/78] TODO --- .../frontend/js_src/lib/components/WbPlanView/navigator.ts | 1 + .../js_src/lib/components/WbPlanView/uploadPlanBuilder.ts | 1 + 2 files changed, 2 insertions(+) diff --git a/specifyweb/frontend/js_src/lib/components/WbPlanView/navigator.ts b/specifyweb/frontend/js_src/lib/components/WbPlanView/navigator.ts index 096565a51d1..bde18adf492 100644 --- a/specifyweb/frontend/js_src/lib/components/WbPlanView/navigator.ts +++ b/specifyweb/frontend/js_src/lib/components/WbPlanView/navigator.ts @@ -301,6 +301,7 @@ export function getMappingLineData({ ? [[formatToManyIndex(maxMappedElementNumber + 1), commonText.add()]] : []; + // Add prop with tree table name and filter if that prop has been passed commitInstanceData( 'toMany', table, diff --git a/specifyweb/frontend/js_src/lib/components/WbPlanView/uploadPlanBuilder.ts b/specifyweb/frontend/js_src/lib/components/WbPlanView/uploadPlanBuilder.ts index 1d820221071..a2ac6db5df8 100644 --- a/specifyweb/frontend/js_src/lib/components/WbPlanView/uploadPlanBuilder.ts +++ b/specifyweb/frontend/js_src/lib/components/WbPlanView/uploadPlanBuilder.ts @@ -38,6 +38,7 @@ const toTreeRecordRanks = ( ]) ); +// TODO: need to update this to include tree def identifier const toTreeRecordVariety = (lines: RA): TreeRecord => ({ ranks: Object.fromEntries( indexMappings(lines).map(([fullRankName, rankMappedFields]) => [ From c8afd1d2df51d22fde48c4cf96f5b98640b7649e Mon Sep 17 00:00:00 2001 From: Caroline D <108160931+CarolineDenis@users.noreply.github.com> Date: Thu, 11 Jul 2024 11:18:10 -0700 Subject: [PATCH 03/78] List of available taxon trees to choose from when opening WB --- .../lib/components/WbPlanView/Mapper.tsx | 3 ++ .../lib/components/WbPlanView/Wrapped.tsx | 49 ++++++++++++------- .../lib/components/WbPlanView/navigator.ts | 9 +++- 3 files changed, 42 insertions(+), 19 deletions(-) diff --git a/specifyweb/frontend/js_src/lib/components/WbPlanView/Mapper.tsx b/specifyweb/frontend/js_src/lib/components/WbPlanView/Mapper.tsx index 044dbca8ab3..4abd75a58c9 100644 --- a/specifyweb/frontend/js_src/lib/components/WbPlanView/Mapper.tsx +++ b/specifyweb/frontend/js_src/lib/components/WbPlanView/Mapper.tsx @@ -139,6 +139,7 @@ export function Mapper(props: { readonly changesMade: boolean; readonly lines: RA; readonly mustMatchPreferences: IR; + readonly taxonType?: string; }): JSX.Element { const [state, dispatch] = React.useReducer( reducer, @@ -447,6 +448,7 @@ export function Mapper(props: { mustMatchPreferences: state.mustMatchPreferences, generateFieldData: 'all', spec: navigatorSpecs.wbPlanView, + taxonType: props.taxonType, }), customSelectType: 'OPENED_LIST', onChange({ isDoubleClick, ...rest }) { @@ -535,6 +537,7 @@ export function Mapper(props: { mustMatchPreferences: state.mustMatchPreferences, generateFieldData: 'all', spec: navigatorSpecs.wbPlanView, + taxonType: props.taxonType, }), }); diff --git a/specifyweb/frontend/js_src/lib/components/WbPlanView/Wrapped.tsx b/specifyweb/frontend/js_src/lib/components/WbPlanView/Wrapped.tsx index 8ec8c97747b..d83a50f0c28 100644 --- a/specifyweb/frontend/js_src/lib/components/WbPlanView/Wrapped.tsx +++ b/specifyweb/frontend/js_src/lib/components/WbPlanView/Wrapped.tsx @@ -10,23 +10,16 @@ import type { LocalizedString } from 'typesafe-i18n'; import type { State } from 'typesafe-reducer'; import { usePromise } from '../../hooks/useAsyncState'; -import { useCachedState } from '../../hooks/useCachedState'; import { useErrorContext } from '../../hooks/useErrorContext'; import { useLiveState } from '../../hooks/useLiveState'; import { commonText } from '../../localization/common'; -import { resourcesText } from '../../localization/resources'; -import { treeText } from '../../localization/tree'; import { wbText } from '../../localization/workbench'; -import { f } from '../../utils/functools'; -import type { IR, RA } from '../../utils/types'; +import { type IR, type RA, localized } from '../../utils/types'; import { caseInsensitiveHash } from '../../utils/utils'; +import { Ul } from '../Atoms'; import { Button } from '../Atoms/Button'; -import { Select } from '../Atoms/Form'; import type { Tables } from '../DataModel/types'; -import { - getTreeDefinitions, - treeRanksPromise, -} from '../InitialContext/treeRanks'; +import { treeRanksPromise } from '../InitialContext/treeRanks'; import { useTitle } from '../Molecules/AppTitle'; import { Dialog } from '../Molecules/Dialog'; import { ProtectedAction } from '../Permissions/PermissionDenied'; @@ -139,10 +132,27 @@ export function WbPlanView({ const [isTaxonTable, setIsTaxonTable] = React.useState(false); const [treeDefinitions] = usePromise(treeRanksPromise, true); - const definitionsForTree = treeDefinitions + const definitionsForTreeTaxon = treeDefinitions ? caseInsensitiveHash(treeDefinitions, 'Taxon') : undefined; - const definitions = definitionsForTree?.map(({ definition }) => definition); + const definitions = definitionsForTreeTaxon?.map( + ({ definition }) => definition + ); + const [taxonType, setTaxonType] = React.useState(); + const handleTreeType = (taxonTreeDefinitionName: string): void => { + setTaxonType(taxonTreeDefinitionName); + setState({ + type: 'MappingState', + changesMade: true, + baseTableName: 'Taxon', + lines: getLinesFromHeaders({ + headers, + runAutoMapper: true, + baseTableName: 'Taxon', + }), + mustMatchPreferences: {}, + }); + }; const navigate = useNavigate(); return state.type === 'SelectBaseTable' ? ( @@ -183,11 +193,15 @@ export function WbPlanView({ header={wbText.selectTree()} onClose={(): void => setIsTaxonTable(false)} > - {definitions?.map(({ id, name }) => ( - - ))} +
    + {definitions?.map(({ name }) => ( +
  • + handleTreeType(name)}> + {localized(name)} + +
    • + ))} +
)} @@ -198,6 +212,7 @@ export function WbPlanView({ dataset={dataset} lines={state.lines} mustMatchPreferences={state.mustMatchPreferences} + taxonType={taxonType} onChangeBaseTable={(): void => setState({ type: 'SelectBaseTable', diff --git a/specifyweb/frontend/js_src/lib/components/WbPlanView/navigator.ts b/specifyweb/frontend/js_src/lib/components/WbPlanView/navigator.ts index bde18adf492..df92bc94015 100644 --- a/specifyweb/frontend/js_src/lib/components/WbPlanView/navigator.ts +++ b/specifyweb/frontend/js_src/lib/components/WbPlanView/navigator.ts @@ -189,6 +189,7 @@ export function getMappingLineData({ showHiddenFields = false, mustMatchPreferences = {}, spec, + taxonType, }: { readonly baseTableName: keyof Tables; // The mapping path @@ -203,6 +204,7 @@ export function getMappingLineData({ readonly showHiddenFields?: boolean; readonly mustMatchPreferences?: IR; readonly spec: NavigatorSpec; + readonly taxonType?: string; }): RA { if ( process.env.NODE_ENV !== 'production' && @@ -254,7 +256,11 @@ export function getMappingLineData({ customSelectSubtype, defaultValue: internalState.defaultValue, selectLabel: table.label, - fieldsData: Object.fromEntries(filterArray(fieldsData)), + fieldsData: Object.fromEntries( + filterArray(fieldsData) + .filter((field) => field[1].tableTreeDefName === taxonType) + .map(([rawKey, fieldData]) => [rawKey, fieldData]) + ), tableName: table.name, }); @@ -301,7 +307,6 @@ export function getMappingLineData({ ? [[formatToManyIndex(maxMappedElementNumber + 1), commonText.add()]] : []; - // Add prop with tree table name and filter if that prop has been passed commitInstanceData( 'toMany', table, From 8fda5d9485a919d7f6bdecd7d0c473f927525fbb Mon Sep 17 00:00:00 2001 From: Caroline D <108160931+CarolineDenis@users.noreply.github.com> Date: Thu, 11 Jul 2024 13:10:04 -0700 Subject: [PATCH 04/78] comments --- .../js_src/lib/components/WbPlanView/CustomSelectElement.tsx | 1 + specifyweb/frontend/js_src/lib/components/WbPlanView/Mapper.tsx | 1 + 2 files changed, 2 insertions(+) diff --git a/specifyweb/frontend/js_src/lib/components/WbPlanView/CustomSelectElement.tsx b/specifyweb/frontend/js_src/lib/components/WbPlanView/CustomSelectElement.tsx index ecab366fb66..f672cb60b52 100644 --- a/specifyweb/frontend/js_src/lib/components/WbPlanView/CustomSelectElement.tsx +++ b/specifyweb/frontend/js_src/lib/components/WbPlanView/CustomSelectElement.tsx @@ -523,6 +523,7 @@ export function CustomSelectElement({ const defaultOption = previewOption ?? + // Where comes from isDefault? inlineOptions.find(({ isDefault }) => isDefault) ?? defaultDefaultOption; diff --git a/specifyweb/frontend/js_src/lib/components/WbPlanView/Mapper.tsx b/specifyweb/frontend/js_src/lib/components/WbPlanView/Mapper.tsx index 4abd75a58c9..e5d82990ab3 100644 --- a/specifyweb/frontend/js_src/lib/components/WbPlanView/Mapper.tsx +++ b/specifyweb/frontend/js_src/lib/components/WbPlanView/Mapper.tsx @@ -458,6 +458,7 @@ export function Mapper(props: { dispatch({ type: 'ChangeSelectElementValueAction', line: 'mappingView', + // TableName in rest is undefined and therefore no mapping ...rest, }); }, From 69cf926e33f1a7ac057a88efa50f3a5b9d01f210 Mon Sep 17 00:00:00 2001 From: Caroline D <108160931+CarolineDenis@users.noreply.github.com> Date: Thu, 11 Jul 2024 13:52:48 -0700 Subject: [PATCH 05/78] Remove comment --- .../js_src/lib/components/WbPlanView/CustomSelectElement.tsx | 1 - specifyweb/frontend/js_src/lib/components/WbPlanView/Mapper.tsx | 1 - 2 files changed, 2 deletions(-) diff --git a/specifyweb/frontend/js_src/lib/components/WbPlanView/CustomSelectElement.tsx b/specifyweb/frontend/js_src/lib/components/WbPlanView/CustomSelectElement.tsx index f672cb60b52..ecab366fb66 100644 --- a/specifyweb/frontend/js_src/lib/components/WbPlanView/CustomSelectElement.tsx +++ b/specifyweb/frontend/js_src/lib/components/WbPlanView/CustomSelectElement.tsx @@ -523,7 +523,6 @@ export function CustomSelectElement({ const defaultOption = previewOption ?? - // Where comes from isDefault? inlineOptions.find(({ isDefault }) => isDefault) ?? defaultDefaultOption; diff --git a/specifyweb/frontend/js_src/lib/components/WbPlanView/Mapper.tsx b/specifyweb/frontend/js_src/lib/components/WbPlanView/Mapper.tsx index e5d82990ab3..4abd75a58c9 100644 --- a/specifyweb/frontend/js_src/lib/components/WbPlanView/Mapper.tsx +++ b/specifyweb/frontend/js_src/lib/components/WbPlanView/Mapper.tsx @@ -458,7 +458,6 @@ export function Mapper(props: { dispatch({ type: 'ChangeSelectElementValueAction', line: 'mappingView', - // TableName in rest is undefined and therefore no mapping ...rest, }); }, From 01c8bd4e275ed80f94b2d01b05297580d6ef5bfd Mon Sep 17 00:00:00 2001 From: Caroline D <108160931+CarolineDenis@users.noreply.github.com> Date: Fri, 12 Jul 2024 07:46:33 -0700 Subject: [PATCH 06/78] Don't filter fieldsData if not first call --- .../frontend/js_src/lib/components/WbPlanView/navigator.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/specifyweb/frontend/js_src/lib/components/WbPlanView/navigator.ts b/specifyweb/frontend/js_src/lib/components/WbPlanView/navigator.ts index df92bc94015..1e5bf580948 100644 --- a/specifyweb/frontend/js_src/lib/components/WbPlanView/navigator.ts +++ b/specifyweb/frontend/js_src/lib/components/WbPlanView/navigator.ts @@ -258,7 +258,12 @@ export function getMappingLineData({ selectLabel: table.label, fieldsData: Object.fromEntries( filterArray(fieldsData) - .filter((field) => field[1].tableTreeDefName === taxonType) + .filter((field) => { + if (internalState.mappingLineData.length === 0) { + return field[1].tableTreeDefName === taxonType; + } + return true; + }) .map(([rawKey, fieldData]) => [rawKey, fieldData]) ), tableName: table.name, From 53f0add4b26b34be013f4e5cf375ff184faeee2f Mon Sep 17 00:00:00 2001 From: Caroline D <108160931+CarolineDenis@users.noreply.github.com> Date: Fri, 12 Jul 2024 09:35:46 -0700 Subject: [PATCH 07/78] Pass taxonTreeId with upload plan --- .../frontend/js_src/lib/components/WbPlanView/Wrapped.tsx | 5 +++++ .../frontend/js_src/lib/components/WbPlanView/helpers.ts | 3 +++ .../frontend/js_src/lib/components/WorkBench/index.tsx | 5 ++++- 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/specifyweb/frontend/js_src/lib/components/WbPlanView/Wrapped.tsx b/specifyweb/frontend/js_src/lib/components/WbPlanView/Wrapped.tsx index d83a50f0c28..546003be0ac 100644 --- a/specifyweb/frontend/js_src/lib/components/WbPlanView/Wrapped.tsx +++ b/specifyweb/frontend/js_src/lib/components/WbPlanView/Wrapped.tsx @@ -153,6 +153,10 @@ export function WbPlanView({ mustMatchPreferences: {}, }); }; + const taxonTree = definitionsForTreeTaxon?.find( + (tree) => tree.definition.name === taxonType + ); + const taxonTreeId = taxonTree?.definition.id; const navigate = useNavigate(); return state.type === 'SelectBaseTable' ? ( @@ -224,6 +228,7 @@ export function WbPlanView({ baseTableName: state.baseTableName, lines, mustMatchPreferences, + taxonTreeId, }).then(() => navigate(`/specify/workbench/${dataset.id}/`)) } /> diff --git a/specifyweb/frontend/js_src/lib/components/WbPlanView/helpers.ts b/specifyweb/frontend/js_src/lib/components/WbPlanView/helpers.ts index 7b7dd213d92..deecea572b1 100644 --- a/specifyweb/frontend/js_src/lib/components/WbPlanView/helpers.ts +++ b/specifyweb/frontend/js_src/lib/components/WbPlanView/helpers.ts @@ -42,11 +42,13 @@ export async function savePlan({ baseTableName, lines, mustMatchPreferences, + taxonTreeId, }: { readonly dataset: Dataset; readonly baseTableName: keyof Tables; readonly lines: RA; readonly mustMatchPreferences: IR; + readonly taxonTreeId?: number | undefined; }): Promise { const renamedLines = renameNewlyCreatedHeaders( baseTableName, @@ -75,6 +77,7 @@ export async function savePlan({ method: 'PUT', body: { uploadplan: uploadPlan, + taxonTreeId, }, }).then(async () => newlyAddedHeaders.length === 0 diff --git a/specifyweb/frontend/js_src/lib/components/WorkBench/index.tsx b/specifyweb/frontend/js_src/lib/components/WorkBench/index.tsx index 5bb210a296f..63cb2684778 100644 --- a/specifyweb/frontend/js_src/lib/components/WorkBench/index.tsx +++ b/specifyweb/frontend/js_src/lib/components/WorkBench/index.tsx @@ -68,4 +68,7 @@ function useDataset( const fetchDataset = async (datasetId: number): Promise => ajax(`/api/workbench/dataset/${datasetId}/`, { headers: { Accept: 'application/json' }, - }).then(({ data }) => data); + }).then(({ data }) => { + console.log(''); + return data; + }); From 3282d1de2ac8c22c6d98476e143f436c2ce82265 Mon Sep 17 00:00:00 2001 From: Caroline D <108160931+CarolineDenis@users.noreply.github.com> Date: Fri, 12 Jul 2024 09:41:00 -0700 Subject: [PATCH 08/78] Add taxonId to uplaodPlan --- .../frontend/js_src/lib/components/WbPlanView/helpers.ts | 5 +++-- .../js_src/lib/components/WbPlanView/uploadPlanBuilder.ts | 4 +++- .../js_src/lib/components/WbPlanView/uploadPlanParser.ts | 1 + 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/specifyweb/frontend/js_src/lib/components/WbPlanView/helpers.ts b/specifyweb/frontend/js_src/lib/components/WbPlanView/helpers.ts index deecea572b1..11adc2b2c65 100644 --- a/specifyweb/frontend/js_src/lib/components/WbPlanView/helpers.ts +++ b/specifyweb/frontend/js_src/lib/components/WbPlanView/helpers.ts @@ -68,7 +68,8 @@ export async function savePlan({ const uploadPlan = uploadPlanBuilder( baseTableName, renamedLines, - getMustMatchTables({ baseTableName, lines, mustMatchPreferences }) + getMustMatchTables({ baseTableName, lines, mustMatchPreferences }), + taxonTreeId ); const dataSetRequestUrl = `/api/workbench/dataset/${dataset.id}/`; @@ -77,7 +78,7 @@ export async function savePlan({ method: 'PUT', body: { uploadplan: uploadPlan, - taxonTreeId, + // TaxonTreeId, }, }).then(async () => newlyAddedHeaders.length === 0 diff --git a/specifyweb/frontend/js_src/lib/components/WbPlanView/uploadPlanBuilder.ts b/specifyweb/frontend/js_src/lib/components/WbPlanView/uploadPlanBuilder.ts index a2ac6db5df8..ede50bf503d 100644 --- a/specifyweb/frontend/js_src/lib/components/WbPlanView/uploadPlanBuilder.ts +++ b/specifyweb/frontend/js_src/lib/components/WbPlanView/uploadPlanBuilder.ts @@ -138,7 +138,8 @@ const toUploadable = ( export const uploadPlanBuilder = ( baseTableName: keyof Tables, lines: RA, - mustMatchPreferences: RR + mustMatchPreferences: RR, + taxonTreeId: number | undefined ): UploadPlan => ({ baseTableName: toLowerCase(baseTableName), uploadable: toUploadable( @@ -149,6 +150,7 @@ export const uploadPlanBuilder = ( .map(([tableName]) => tableName), true ), + taxonTreeId, }); const indexMappings = ( diff --git a/specifyweb/frontend/js_src/lib/components/WbPlanView/uploadPlanParser.ts b/specifyweb/frontend/js_src/lib/components/WbPlanView/uploadPlanParser.ts index 3a9463ca621..4fc801073c7 100644 --- a/specifyweb/frontend/js_src/lib/components/WbPlanView/uploadPlanParser.ts +++ b/specifyweb/frontend/js_src/lib/components/WbPlanView/uploadPlanParser.ts @@ -46,6 +46,7 @@ export type Uploadable = TreeRecordVariety | UploadTableVariety; export type UploadPlan = { readonly baseTableName: Lowercase; readonly uploadable: Uploadable; + readonly taxonTreeId: number | undefined; }; const parseColumnOptions = (matchingOptions: ColumnOptions): ColumnOptions => ({ From 659304fe612705410f3d339a3d47daf0e063aa0f Mon Sep 17 00:00:00 2001 From: Caroline D <108160931+CarolineDenis@users.noreply.github.com> Date: Fri, 12 Jul 2024 13:33:38 -0700 Subject: [PATCH 09/78] Use treeDefId to define which tree to look at in mapper --- .../frontend/js_src/lib/components/WbPlanView/Wrapped.tsx | 7 ++++++- .../frontend/js_src/lib/components/WorkBench/index.tsx | 5 +---- specifyweb/workbench/views.py | 4 +++- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/specifyweb/frontend/js_src/lib/components/WbPlanView/Wrapped.tsx b/specifyweb/frontend/js_src/lib/components/WbPlanView/Wrapped.tsx index 546003be0ac..554a8f68326 100644 --- a/specifyweb/frontend/js_src/lib/components/WbPlanView/Wrapped.tsx +++ b/specifyweb/frontend/js_src/lib/components/WbPlanView/Wrapped.tsx @@ -138,6 +138,7 @@ export function WbPlanView({ const definitions = definitionsForTreeTaxon?.map( ({ definition }) => definition ); + const [taxonType, setTaxonType] = React.useState(); const handleTreeType = (taxonTreeDefinitionName: string): void => { setTaxonType(taxonTreeDefinitionName); @@ -157,6 +158,10 @@ export function WbPlanView({ (tree) => tree.definition.name === taxonType ); const taxonTreeId = taxonTree?.definition.id; + const taxonCorrespondingId = definitionsForTreeTaxon?.find( + (tree) => tree.definition.id === uploadPlan?.taxonTreeId + ); + const taxonIdName = taxonCorrespondingId?.definition.name; const navigate = useNavigate(); return state.type === 'SelectBaseTable' ? ( @@ -216,7 +221,7 @@ export function WbPlanView({ dataset={dataset} lines={state.lines} mustMatchPreferences={state.mustMatchPreferences} - taxonType={taxonType} + taxonType={taxonType ?? taxonIdName} onChangeBaseTable={(): void => setState({ type: 'SelectBaseTable', diff --git a/specifyweb/frontend/js_src/lib/components/WorkBench/index.tsx b/specifyweb/frontend/js_src/lib/components/WorkBench/index.tsx index 63cb2684778..5bb210a296f 100644 --- a/specifyweb/frontend/js_src/lib/components/WorkBench/index.tsx +++ b/specifyweb/frontend/js_src/lib/components/WorkBench/index.tsx @@ -68,7 +68,4 @@ function useDataset( const fetchDataset = async (datasetId: number): Promise => ajax(`/api/workbench/dataset/${datasetId}/`, { headers: { Accept: 'application/json' }, - }).then(({ data }) => { - console.log(''); - return data; - }); + }).then(({ data }) => data); diff --git a/specifyweb/workbench/views.py b/specifyweb/workbench/views.py index ead475776e4..b77dc83a1ee 100644 --- a/specifyweb/workbench/views.py +++ b/specifyweb/workbench/views.py @@ -477,7 +477,9 @@ def dataset(request, ds: models.Spdataset) -> http.HttpResponse: try: validate(plan, upload_plan_schema.schema) except ValidationError as e: - return http.HttpResponse(f"upload plan is invalid: {e}", status=400) + # TODO fix this + # return http.HttpResponse(f"upload plan is invalid: {e}", status=400) + pass new_cols = upload_plan_schema.parse_plan(request.specify_collection, plan).get_cols() - set(ds.columns) if new_cols: From eeac2333479bc7b69c0dc1b63ad66ec8c68d7bb7 Mon Sep 17 00:00:00 2001 From: Caroline D <108160931+CarolineDenis@users.noreply.github.com> Date: Tue, 16 Jul 2024 09:06:42 -0700 Subject: [PATCH 10/78] Remove comment --- specifyweb/frontend/js_src/lib/components/WbPlanView/helpers.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/specifyweb/frontend/js_src/lib/components/WbPlanView/helpers.ts b/specifyweb/frontend/js_src/lib/components/WbPlanView/helpers.ts index 11adc2b2c65..af69d9667e7 100644 --- a/specifyweb/frontend/js_src/lib/components/WbPlanView/helpers.ts +++ b/specifyweb/frontend/js_src/lib/components/WbPlanView/helpers.ts @@ -78,7 +78,6 @@ export async function savePlan({ method: 'PUT', body: { uploadplan: uploadPlan, - // TaxonTreeId, }, }).then(async () => newlyAddedHeaders.length === 0 From dd675ba98a8f39d08bd040fa2ece30f04d494632 Mon Sep 17 00:00:00 2001 From: Caroline D <108160931+CarolineDenis@users.noreply.github.com> Date: Tue, 16 Jul 2024 09:47:35 -0700 Subject: [PATCH 11/78] Add treeId to upload plan --- specifyweb/workbench/upload/upload_plan_schema.py | 13 +++++++++++++ specifyweb/workbench/views.py | 4 +--- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/specifyweb/workbench/upload/upload_plan_schema.py b/specifyweb/workbench/upload/upload_plan_schema.py index 3826fbf4263..e617036d962 100644 --- a/specifyweb/workbench/upload/upload_plan_schema.py +++ b/specifyweb/workbench/upload/upload_plan_schema.py @@ -34,6 +34,12 @@ }, ] }, + "taxonTreeId": { + "oneOf": [ + { "type": "integer" }, + { "type": "null" } + ] + } }, 'required': [ 'baseTableName', 'uploadable' ], 'additionalProperties': False, @@ -140,6 +146,13 @@ ], }, + 'taxonTreeId': { + "oneOf": [ + { "type": "integer" }, + { "type": "null" } + ] + }, + 'wbcols': { 'type': 'object', 'description': 'Maps the columns of the destination table to the headers of the source columns of input data.', diff --git a/specifyweb/workbench/views.py b/specifyweb/workbench/views.py index b77dc83a1ee..ead475776e4 100644 --- a/specifyweb/workbench/views.py +++ b/specifyweb/workbench/views.py @@ -477,9 +477,7 @@ def dataset(request, ds: models.Spdataset) -> http.HttpResponse: try: validate(plan, upload_plan_schema.schema) except ValidationError as e: - # TODO fix this - # return http.HttpResponse(f"upload plan is invalid: {e}", status=400) - pass + return http.HttpResponse(f"upload plan is invalid: {e}", status=400) new_cols = upload_plan_schema.parse_plan(request.specify_collection, plan).get_cols() - set(ds.columns) if new_cols: From 670fc872ffe8c1d9306a7db5cb6f5d7d046bdf1d Mon Sep 17 00:00:00 2001 From: Caroline D <108160931+CarolineDenis@users.noreply.github.com> Date: Tue, 16 Jul 2024 10:44:51 -0700 Subject: [PATCH 12/78] Fix filtering when taxonType is undefined, needed in QB --- .../frontend/js_src/lib/components/WbPlanView/navigator.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/specifyweb/frontend/js_src/lib/components/WbPlanView/navigator.ts b/specifyweb/frontend/js_src/lib/components/WbPlanView/navigator.ts index 1e5bf580948..f67123204d5 100644 --- a/specifyweb/frontend/js_src/lib/components/WbPlanView/navigator.ts +++ b/specifyweb/frontend/js_src/lib/components/WbPlanView/navigator.ts @@ -259,7 +259,10 @@ export function getMappingLineData({ fieldsData: Object.fromEntries( filterArray(fieldsData) .filter((field) => { - if (internalState.mappingLineData.length === 0) { + if ( + internalState.mappingLineData.length === 0 && + taxonType !== undefined + ) { return field[1].tableTreeDefName === taxonType; } return true; From e7e83aeebb06f69f0faf34cff52e7e14b576bdbc Mon Sep 17 00:00:00 2001 From: Caroline D <108160931+CarolineDenis@users.noreply.github.com> Date: Fri, 2 Aug 2024 17:37:37 -0500 Subject: [PATCH 13/78] Start work on adding new column in WB for multiple trees --- .../lib/components/WbPlanView/Mapper.tsx | 9 ++++ .../lib/components/WbPlanView/Wrapped.tsx | 15 ++++--- .../lib/components/WbPlanView/navigator.ts | 42 +++++++++++++++---- 3 files changed, 53 insertions(+), 13 deletions(-) diff --git a/specifyweb/frontend/js_src/lib/components/WbPlanView/Mapper.tsx b/specifyweb/frontend/js_src/lib/components/WbPlanView/Mapper.tsx index 4abd75a58c9..ef6d2a874e9 100644 --- a/specifyweb/frontend/js_src/lib/components/WbPlanView/Mapper.tsx +++ b/specifyweb/frontend/js_src/lib/components/WbPlanView/Mapper.tsx @@ -24,10 +24,15 @@ import { className } from '../Atoms/className'; import { icons } from '../Atoms/Icons'; import { Link } from '../Atoms/Link'; import { LoadingContext, ReadOnlyContext } from '../Core/Contexts'; +import type { + FilterTablesByEndsWith, + SerializedResource, +} from '../DataModel/helperTypes'; import { strictGetTable } from '../DataModel/tables'; import type { Tables } from '../DataModel/types'; import { softFail } from '../Errors/Crash'; import { ErrorBoundary } from '../Errors/ErrorBoundary'; +import type { TreeInformation } from '../InitialContext/treeRanks'; import { TableIcon } from '../Molecules/TableIcon'; import { Layout } from './Header'; import { @@ -140,6 +145,9 @@ export function Mapper(props: { readonly lines: RA; readonly mustMatchPreferences: IR; readonly taxonType?: string; + readonly treeDefinitions: + | readonly SerializedResource>[] + | undefined; }): JSX.Element { const [state, dispatch] = React.useReducer( reducer, @@ -449,6 +457,7 @@ export function Mapper(props: { generateFieldData: 'all', spec: navigatorSpecs.wbPlanView, taxonType: props.taxonType, + treeDefinitions: props.treeDefinitions, }), customSelectType: 'OPENED_LIST', onChange({ isDoubleClick, ...rest }) { diff --git a/specifyweb/frontend/js_src/lib/components/WbPlanView/Wrapped.tsx b/specifyweb/frontend/js_src/lib/components/WbPlanView/Wrapped.tsx index 554a8f68326..2821282505f 100644 --- a/specifyweb/frontend/js_src/lib/components/WbPlanView/Wrapped.tsx +++ b/specifyweb/frontend/js_src/lib/components/WbPlanView/Wrapped.tsx @@ -169,10 +169,13 @@ export function WbPlanView({ navigate(`/specify/workbench/${dataset.id}/`)} - onSelected={(baseTableName): void => { - if (baseTableName === 'Taxon') { - setIsTaxonTable(true); - } else { + onSelected={ + (baseTableName): void => { + /* + * If (baseTableName === 'Taxon') { + * setIsTaxonTable(true); + * } else { + */ setState({ type: 'MappingState', changesMade: true, @@ -185,7 +188,8 @@ export function WbPlanView({ mustMatchPreferences: {}, }); } - }} + // } + } onSelectTemplate={(uploadPlan, headers): void => setState({ type: 'MappingState', @@ -222,6 +226,7 @@ export function WbPlanView({ lines={state.lines} mustMatchPreferences={state.mustMatchPreferences} taxonType={taxonType ?? taxonIdName} + treeDefinitions={definitions} onChangeBaseTable={(): void => setState({ type: 'SelectBaseTable', diff --git a/specifyweb/frontend/js_src/lib/components/WbPlanView/navigator.ts b/specifyweb/frontend/js_src/lib/components/WbPlanView/navigator.ts index 8911ed58e67..e3cba7a6676 100644 --- a/specifyweb/frontend/js_src/lib/components/WbPlanView/navigator.ts +++ b/specifyweb/frontend/js_src/lib/components/WbPlanView/navigator.ts @@ -10,11 +10,16 @@ import { queryText } from '../../localization/query'; import type { IR, RA, WritableArray } from '../../utils/types'; import { defined, filterArray } from '../../utils/types'; import { dateParts } from '../Atoms/Internationalization'; -import type { AnyTree } from '../DataModel/helperTypes'; +import type { + AnyTree, + FilterTablesByEndsWith, + SerializedResource, +} from '../DataModel/helperTypes'; import type { Relationship } from '../DataModel/specifyField'; import type { SpecifyTable } from '../DataModel/specifyTable'; import { getFrontEndOnlyFields, strictGetTable } from '../DataModel/tables'; import type { Tables } from '../DataModel/types'; +import type { TreeInformation } from '../InitialContext/treeRanks'; import { getTreeDefinitions, isTreeTable } from '../InitialContext/treeRanks'; import { hasTablePermission, hasTreeAccess } from '../Permissions/helpers'; import type { CustomSelectSubtype } from './CustomSelectElement'; @@ -195,6 +200,7 @@ export function getMappingLineData({ mustMatchPreferences = {}, spec, taxonType, + treeDefinitions, }: { readonly baseTableName: keyof Tables; // The mapping path @@ -210,6 +216,9 @@ export function getMappingLineData({ readonly mustMatchPreferences?: IR; readonly spec: NavigatorSpec; readonly taxonType?: string; + readonly treeDefinitions: + | readonly SerializedResource>[] + | undefined; }): RA { if ( process.env.NODE_ENV !== 'production' && @@ -264,7 +273,9 @@ export function getMappingLineData({ fieldsData: Object.fromEntries( filterArray(fieldsData) .filter((field) => { - if ( + if (table.name === 'Taxon' && mappingPath[0] === '0') { + return treeDefinitions?.map((definition) => definition.name); + } else if ( internalState.mappingLineData.length === 0 && taxonType !== undefined ) { @@ -350,12 +361,24 @@ export function getMappingLineData({ handleTreeRanks({ table }) { const defaultValue = getNameFromTreeRankName(internalState.defaultValue); - commitInstanceData( - 'tree', - table, + const fieldsData: readonly ( + | readonly [string, HtmlGeneratorFieldData] + | undefined + )[] = generateFieldData === 'none' || - !hasTreeAccess(table.name as 'Geography', 'read') + !hasTreeAccess(table.name as 'Geography', 'read') ? [] + : table.name === 'Taxon' && mappingPath[0] === '0' && treeDefinitions + ? treeDefinitions.map((treeDefinition) => [ + treeDefinition.name, + { + optionLabel: treeDefinition.name, + isRelationship: true, + isDefault: false, + tableName: 'Taxon', + tableTreeDefName: treeDefinition.name, + }, + ]) : [ spec.includeAnyTreeRank && (generateFieldData === 'all' || @@ -395,8 +418,11 @@ export function getMappingLineData({ ) ) : []), - ] - ); + ]; + + console.log(fieldsData); + + commitInstanceData('tree', table, fieldsData); }, handleSimpleFields({ table, parentRelationship }) { From f8dec7fbecf75d4786426a5990c94dfd0f2e886b Mon Sep 17 00:00:00 2001 From: Caroline D <108160931+CarolineDenis@users.noreply.github.com> Date: Mon, 5 Aug 2024 09:11:36 -0700 Subject: [PATCH 14/78] add handleTreeSelection --- .../js_src/lib/components/WbPlanView/Mapper.tsx | 3 +++ .../js_src/lib/components/WbPlanView/Wrapped.tsx | 13 ++++++++++++- .../js_src/lib/components/WbPlanView/navigator.ts | 13 ++++++++----- 3 files changed, 23 insertions(+), 6 deletions(-) diff --git a/specifyweb/frontend/js_src/lib/components/WbPlanView/Mapper.tsx b/specifyweb/frontend/js_src/lib/components/WbPlanView/Mapper.tsx index ef6d2a874e9..3125f322e3b 100644 --- a/specifyweb/frontend/js_src/lib/components/WbPlanView/Mapper.tsx +++ b/specifyweb/frontend/js_src/lib/components/WbPlanView/Mapper.tsx @@ -148,6 +148,7 @@ export function Mapper(props: { readonly treeDefinitions: | readonly SerializedResource>[] | undefined; + readonly handleTreeSelection: (treeName: string) => void; }): JSX.Element { const [state, dispatch] = React.useReducer( reducer, @@ -461,6 +462,8 @@ export function Mapper(props: { }), customSelectType: 'OPENED_LIST', onChange({ isDoubleClick, ...rest }) { + if (rest.index === 0 && rest.newTableName === 'Taxon') + props.handleTreeSelection(rest.newValue); if (isDoubleClick && mapButtonEnabled) dispatch({ type: 'MappingViewMapAction' }); else if (!isReadOnly) diff --git a/specifyweb/frontend/js_src/lib/components/WbPlanView/Wrapped.tsx b/specifyweb/frontend/js_src/lib/components/WbPlanView/Wrapped.tsx index 2821282505f..5ffccd52b74 100644 --- a/specifyweb/frontend/js_src/lib/components/WbPlanView/Wrapped.tsx +++ b/specifyweb/frontend/js_src/lib/components/WbPlanView/Wrapped.tsx @@ -163,6 +163,16 @@ export function WbPlanView({ ); const taxonIdName = taxonCorrespondingId?.definition.name; + const [treeSelected, setTreeSelected] = React.useState(); + + const handleTreeSelection = (treeName: string): void => { + const taxonCorresponding = definitionsForTreeTaxon?.find( + (tree) => tree.definition.name === treeName + ); + const taxonIdName = taxonCorresponding?.definition.name; + setTreeSelected(taxonIdName); + }; + const navigate = useNavigate(); return state.type === 'SelectBaseTable' ? ( @@ -223,9 +233,10 @@ export function WbPlanView({ baseTableName={state.baseTableName} changesMade={state.changesMade} dataset={dataset} + handleTreeSelection={handleTreeSelection} lines={state.lines} mustMatchPreferences={state.mustMatchPreferences} - taxonType={taxonType ?? taxonIdName} + taxonType={taxonType ?? treeSelected} treeDefinitions={definitions} onChangeBaseTable={(): void => setState({ diff --git a/specifyweb/frontend/js_src/lib/components/WbPlanView/navigator.ts b/specifyweb/frontend/js_src/lib/components/WbPlanView/navigator.ts index e3cba7a6676..31a02113a02 100644 --- a/specifyweb/frontend/js_src/lib/components/WbPlanView/navigator.ts +++ b/specifyweb/frontend/js_src/lib/components/WbPlanView/navigator.ts @@ -275,12 +275,15 @@ export function getMappingLineData({ .filter((field) => { if (table.name === 'Taxon' && mappingPath[0] === '0') { return treeDefinitions?.map((definition) => definition.name); - } else if ( - internalState.mappingLineData.length === 0 && - taxonType !== undefined - ) { - return field[1].tableTreeDefName === taxonType; } + /* + * Else if ( + * internalState.mappingLineData.length === 0 && + * taxonType !== undefined + * ) { + * return field[1].tableTreeDefName === taxonType; + * } + */ return true; }) .map(([rawKey, fieldData]) => [rawKey, fieldData]) From 22ccb87a3ee93620b434d9d8bba4902d29c60734 Mon Sep 17 00:00:00 2001 From: Caroline D <108160931+CarolineDenis@users.noreply.github.com> Date: Mon, 5 Aug 2024 12:45:20 -0700 Subject: [PATCH 15/78] Chnage variable name (duplicate) --- .../frontend/js_src/lib/components/WbPlanView/navigator.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/specifyweb/frontend/js_src/lib/components/WbPlanView/navigator.ts b/specifyweb/frontend/js_src/lib/components/WbPlanView/navigator.ts index 31a02113a02..341329c0dab 100644 --- a/specifyweb/frontend/js_src/lib/components/WbPlanView/navigator.ts +++ b/specifyweb/frontend/js_src/lib/components/WbPlanView/navigator.ts @@ -364,7 +364,7 @@ export function getMappingLineData({ handleTreeRanks({ table }) { const defaultValue = getNameFromTreeRankName(internalState.defaultValue); - const fieldsData: readonly ( + const fieldsTreeData: readonly ( | readonly [string, HtmlGeneratorFieldData] | undefined )[] = @@ -423,9 +423,7 @@ export function getMappingLineData({ : []), ]; - console.log(fieldsData); - - commitInstanceData('tree', table, fieldsData); + commitInstanceData('tree', table, fieldsTreeData); }, handleSimpleFields({ table, parentRelationship }) { From fb477d0d46684d2e31c3151d3c584dcff1272c99 Mon Sep 17 00:00:00 2001 From: alec_dev Date: Tue, 20 Aug 2024 09:47:23 -0500 Subject: [PATCH 16/78] init wb uploads handling taxon ranks with treedefid --- specifyweb/workbench/upload/scoping.py | 3 +- specifyweb/workbench/upload/treerecord.py | 52 ++++++++++-- .../workbench/upload/upload_plan_schema.py | 83 +++++++++++++++++-- specifyweb/workbench/views.py | 10 ++- 4 files changed, 132 insertions(+), 16 deletions(-) diff --git a/specifyweb/workbench/upload/scoping.py b/specifyweb/workbench/upload/scoping.py index 2d84a98a8ba..07e5213cc12 100644 --- a/specifyweb/workbench/upload/scoping.py +++ b/specifyweb/workbench/upload/scoping.py @@ -188,8 +188,7 @@ def apply_scoping_to_treerecord(tr: TreeRecord, collection) -> ScopedTreeRecord: if table.name == 'Taxon': # TODO: Add scoping in the workbench via the Upload Plan - # treedef = get_taxon_treedef(collection) - treedef = collection.discipline.taxontreedef + treedef = models.Taxontreedef.objects.filter(id=tr.treedef_id).first() elif table.name == 'Geography': treedef = collection.discipline.geographytreedef diff --git a/specifyweb/workbench/upload/treerecord.py b/specifyweb/workbench/upload/treerecord.py index d34f7619aa5..5f7a71b4c2f 100644 --- a/specifyweb/workbench/upload/treerecord.py +++ b/specifyweb/workbench/upload/treerecord.py @@ -23,25 +23,48 @@ class TreeRecord(NamedTuple): name: str ranks: Dict[str, Dict[str, ColumnOptions]] + # treedef_id: int def apply_scoping(self, collection) -> "ScopedTreeRecord": from .scoping import apply_scoping_to_treerecord as apply_scoping return apply_scoping(self, collection) def get_cols(self) -> Set[str]: - return set(col.column for r in self.ranks.values() for col in r.values()) + return {col.column for r in self.ranks.values() for col in r.values() if hasattr(col, 'column')} def to_json(self) -> Dict: result = { - 'ranks': { - rank: cols['name'].to_json() if len(cols) == 1 else dict(treeNodeCols={k: v.to_json() for k, v in cols.items()}) + "ranks": { + rank: ( + cols["name"].to_json() + if len(cols) == 1 + else dict( + treeNodeCols={ + k: v.to_json() if hasattr(v, "to_json") else v + for k, v in cols.items() + } + ) + ) for rank, cols in self.ranks.items() }, } - return { 'treeRecord': result } + return {'treeRecord': result} + + # def to_json(self) -> Dict: + # result = {'ranks': {}} + # for rank, cols in self.ranks.items(): + # if len(cols) == 1: + # result['ranks'][rank] = cols['name'].to_json() + # else: + # treeNodeCols = {} + # for k, v in cols.items(): + # treeNodeCols[k] = v.to_json() if hasattr(v, 'to_json') else v + # result['ranks'][rank] = {'treeNodeCols': treeNodeCols} + # return {'treeRecord': result} def unparse(self) -> Dict: - return { 'baseTableName': self.name, 'uploadble': self.to_json() } + return { 'baseTableName': self.name, + 'uploadble': self.to_json() } class ScopedTreeRecord(NamedTuple): name: str @@ -50,6 +73,7 @@ class ScopedTreeRecord(NamedTuple): treedefitems: List root: Optional[Any] disambiguation: Dict[str, int] + ranks_treedefid_map: Dict[str, int] def disambiguate(self, disambiguation: DA) -> "ScopedTreeRecord": return self._replace(disambiguation=disambiguation.disambiguate_tree()) if disambiguation is not None else self @@ -87,6 +111,24 @@ def bind(self, collection, row: Row, uploadingAgentId: Optional[int], auditor: A auditor=auditor, cache=cache, ) + + def get_cols(self) -> Set[str]: + return set(col.column for r in self.ranks.values() for col in r.values()) + + # def apply_scoping(self, collection) -> "ScopedTreeRecord": + # return self + + # def to_json(self) -> Dict: + # result = { + # 'ranks': { + # rank: cols['name'].to_json() if len(cols) == 1 else dict(treeNodeCols={k: v.to_json() for k, v in cols.items()}) + # for rank, cols in self.ranks.items() + # }, + # } + # return { 'treeRecord': result } + + # def unparse(self) -> Dict: + # return { 'baseTableName': self.name, 'uploadble': self.to_json() } class MustMatchTreeRecord(TreeRecord): def apply_scoping(self, collection) -> "ScopedMustMatchTreeRecord": diff --git a/specifyweb/workbench/upload/upload_plan_schema.py b/specifyweb/workbench/upload/upload_plan_schema.py index e617036d962..8cd9c1fdb7c 100644 --- a/specifyweb/workbench/upload/upload_plan_schema.py +++ b/specifyweb/workbench/upload/upload_plan_schema.py @@ -1,16 +1,19 @@ -from typing import Dict, Any, Union, Tuple +from os import name +from typing import Dict, Any, Optional, Union, Tuple +import logging from specifyweb.specify.datamodel import datamodel, Table, Relationship from specifyweb.specify.load_datamodel import DoesNotExistError -from specifyweb.specify import models +from specifyweb.specify.models import Taxontreedefitem, Taxontreedef from .upload_table import DeferredScopeUploadTable, UploadTable, OneToOneTable, MustMatchTable from .tomany import ToManyRecord -from .treerecord import TreeRecord, MustMatchTreeRecord +from .treerecord import TreeRecord, ScopedTreeRecord, MustMatchTreeRecord from .uploadable import Uploadable from .column_options import ColumnOptions from .scoping import DEFERRED_SCOPING +logger = logging.getLogger(__name__) schema: Dict = { 'title': 'Specify 7 Workbench Upload Plan', @@ -228,14 +231,14 @@ } -def parse_plan_with_basetable(collection, to_parse: Dict) -> Tuple[Table, Uploadable]: +def parse_plan_with_basetable(collection, to_parse: Dict, extra: Dict = {}) -> Tuple[Table, Uploadable]: base_table = datamodel.get_table_strict(to_parse['baseTableName']) return base_table, parse_uploadable(collection, base_table, to_parse['uploadable']) -def parse_plan(collection, to_parse: Dict) -> Uploadable: - return parse_plan_with_basetable(collection, to_parse)[1] +def parse_plan(collection, to_parse: Dict, extra: Dict = {}) -> Uploadable: + return parse_plan_with_basetable(collection, to_parse, extra)[1] -def parse_uploadable(collection, table: Table, to_parse: Dict) -> Uploadable: +def parse_uploadable(collection, table: Table, to_parse: Dict, extra: Dict = {}) -> Uploadable: if 'uploadTable' in to_parse: return parse_upload_table(collection, table, to_parse['uploadTable']) @@ -249,7 +252,10 @@ def parse_uploadable(collection, table: Table, to_parse: Dict) -> Uploadable: return MustMatchTreeRecord(*parse_tree_record(collection, table, to_parse['mustMatchTreeRecord'])) if 'treeRecord' in to_parse: - return parse_tree_record(collection, table, to_parse['treeRecord']) + treedefid = None + if 'treedefid' in to_parse.keys(): + treedefid = int(to_parse['treedefid']) + return parse_tree_record(collection, table, to_parse['treeRecord'], treedefid) raise ValueError('unknown uploadable type') @@ -314,6 +320,67 @@ def parse_tree_record(collection, table: Table, to_parse: Dict) -> TreeRecord: ranks=ranks, ) +def parse_tree_record_2(collection, table: Table, to_parse: Dict, base_treedef_id: Optional[int] = None) -> TreeRecord: + ranks = {} + ranks_treedefid_map = {} + treedefitems = [] + for rank, name_or_cols in to_parse['ranks'].items(): + if isinstance(name_or_cols, str): + ranks[rank] = {'name': parse_column_options(name_or_cols)} + else: + ranks[rank] = {} + for k, v in name_or_cols['treeNodeCols'].items(): + ranks[rank][k] = parse_column_options(v) + if 'treedefid' in name_or_cols.keys(): + treedef_id = int(name_or_cols['treedefid']) + ranks_treedefid_map[rank] = treedef_id + treedefitems.append(Taxontreedefitem.objects.filter(treedef_id=treedef_id, name=rank).first()) + ranks[rank]['treedefid'] = treedef_id + elif 'treedefname' in name_or_cols.keys(): + treedef_id = Taxontreedef.objects.filter(name=name_or_cols['treedefname']).first().id + ranks[rank]['treedefid'] = str(treedef_id) + else: + taxontreedefitems = Taxontreedefitem.objects.filter(name=rank) + # if taxontreedefitems.count() > 1: + # raise ValueError(f"Multiple treedefitems with name {rank}") + treedefitem = taxontreedefitems.first() + # if treedefitem is None: + # raise ValueError(f"Could not find treedefitem with name {rank}") + treedef_id = treedefitem.treedef_id + ranks[rank]['treedefid'] = str(treedef_id) + # TODO: Handle cases where two ranks have the same name but different treedefids + + for rank, cols in ranks.items(): + assert 'name' in cols, to_parse + + return TreeRecord( + name=table.django_name, + ranks=ranks + ) + +def adjust_upload_plan(plan: Dict) -> Dict: + def get_treedef_id(rank_name: str, base_treedef_id: Optional[int]) -> int: + if base_treedef_id is not None: + treedef = Taxontreedefitem.objects.filter(treedef_id=base_treedef_id, name=rank_name).first() + if treedef: + if treedef.count() > 1: + raise ValueError(f"Multiple treedefitems with name {rank_name} and treedef_id {base_treedef_id}") + return base_treedef_id + treedef = Taxontreedefitem.objects.filter(name=rank_name).first() + if treedef: + return treedef.id + raise DoesNotExistError(f"Could not find treedefitem with name {rank_name}") + + if plan['baseTableName'] == 'taxon': + base_treedef_id = plan.get('taxonTreeId', None) + for rank, name_or_cols in plan['uploadable']['treeRecord']['ranks'].items(): + if isinstance(name_or_cols, dict): + rank_name = name_or_cols['treeNodeCols']['name'] + if 'treedefid' not in name_or_cols: + name_or_cols['treedefid'] = get_treedef_id(rank_name, base_treedef_id) + + return plan + def parse_to_many_record(collection, table: Table, to_parse: Dict) -> ToManyRecord: def rel_table(key: str) -> Table: diff --git a/specifyweb/workbench/views.py b/specifyweb/workbench/views.py index b77dc83a1ee..f3a271bd59a 100644 --- a/specifyweb/workbench/views.py +++ b/specifyweb/workbench/views.py @@ -481,7 +481,15 @@ def dataset(request, ds: models.Spdataset) -> http.HttpResponse: # return http.HttpResponse(f"upload plan is invalid: {e}", status=400) pass - new_cols = upload_plan_schema.parse_plan(request.specify_collection, plan).get_cols() - set(ds.columns) + extra = {} + if 'taxonTreeId' in plan.keys(): + extra['treedefid'] = plan['taxonTreeId'] + + parsed_plan = upload_plan_schema.parse_plan(request.specify_collection, plan, extra) + new_cols = parsed_plan.get_cols() - set(ds.columns) + if plan['baseTableName'] == 'taxon': + # plan['uploadable'] = parsed_plan.to_json() + plan = upload_plan_schema.adjust_upload_plan(plan) if new_cols: ncols = len(ds.columns) ds.columns += list(new_cols) From 4b12da588b86969ce768e0fa03bb7305389be11f Mon Sep 17 00:00:00 2001 From: alec_dev Date: Tue, 20 Aug 2024 11:07:44 -0500 Subject: [PATCH 17/78] simplify wb mutli-tree solution --- specifyweb/workbench/upload/scoping.py | 11 ++- specifyweb/workbench/upload/treerecord.py | 33 +------ .../workbench/upload/upload_plan_schema.py | 94 ++++++++----------- specifyweb/workbench/views.py | 7 +- 4 files changed, 47 insertions(+), 98 deletions(-) diff --git a/specifyweb/workbench/upload/scoping.py b/specifyweb/workbench/upload/scoping.py index 07e5213cc12..7815fba9489 100644 --- a/specifyweb/workbench/upload/scoping.py +++ b/specifyweb/workbench/upload/scoping.py @@ -187,8 +187,10 @@ def apply_scoping_to_treerecord(tr: TreeRecord, collection) -> ScopedTreeRecord: table = datamodel.get_table_strict(tr.name) if table.name == 'Taxon': - # TODO: Add scoping in the workbench via the Upload Plan - treedef = models.Taxontreedef.objects.filter(id=tr.treedef_id).first() + if tr.treedef_id is not None: + treedef = models.Taxontreedef.objects.filter(id=tr.treedef_id).first() + else: + treedef = collection.discipline.taxontreedef elif table.name == 'Geography': treedef = collection.discipline.geographytreedef @@ -205,6 +207,9 @@ def apply_scoping_to_treerecord(tr: TreeRecord, collection) -> ScopedTreeRecord: else: raise Exception(f'unexpected tree type: {table.name}') + if treedef is None: + raise ValueError(f"Could not find treedef for table {table.name}") + treedefitems = list(treedef.treedefitems.order_by('rankid')) treedef_ranks = [tdi.name for tdi in treedefitems] for rank in tr.ranks: @@ -217,7 +222,7 @@ def apply_scoping_to_treerecord(tr: TreeRecord, collection) -> ScopedTreeRecord: name=tr.name, ranks={r: {f: extend_columnoptions(colopts, collection, table.name, f) for f, colopts in cols.items()} for r, cols in tr.ranks.items()}, treedef=treedef, - treedefitems=list(treedef.treedefitems.order_by('rankid')), + treedefitems=treedefitems, root=root[0] if root else None, disambiguation={}, ) diff --git a/specifyweb/workbench/upload/treerecord.py b/specifyweb/workbench/upload/treerecord.py index 5f7a71b4c2f..27493fc9d27 100644 --- a/specifyweb/workbench/upload/treerecord.py +++ b/specifyweb/workbench/upload/treerecord.py @@ -23,7 +23,7 @@ class TreeRecord(NamedTuple): name: str ranks: Dict[str, Dict[str, ColumnOptions]] - # treedef_id: int + treedef_id: Optional[int] = None def apply_scoping(self, collection) -> "ScopedTreeRecord": from .scoping import apply_scoping_to_treerecord as apply_scoping @@ -50,18 +50,6 @@ def to_json(self) -> Dict: } return {'treeRecord': result} - # def to_json(self) -> Dict: - # result = {'ranks': {}} - # for rank, cols in self.ranks.items(): - # if len(cols) == 1: - # result['ranks'][rank] = cols['name'].to_json() - # else: - # treeNodeCols = {} - # for k, v in cols.items(): - # treeNodeCols[k] = v.to_json() if hasattr(v, 'to_json') else v - # result['ranks'][rank] = {'treeNodeCols': treeNodeCols} - # return {'treeRecord': result} - def unparse(self) -> Dict: return { 'baseTableName': self.name, 'uploadble': self.to_json() } @@ -73,7 +61,6 @@ class ScopedTreeRecord(NamedTuple): treedefitems: List root: Optional[Any] disambiguation: Dict[str, int] - ranks_treedefid_map: Dict[str, int] def disambiguate(self, disambiguation: DA) -> "ScopedTreeRecord": return self._replace(disambiguation=disambiguation.disambiguate_tree()) if disambiguation is not None else self @@ -111,24 +98,6 @@ def bind(self, collection, row: Row, uploadingAgentId: Optional[int], auditor: A auditor=auditor, cache=cache, ) - - def get_cols(self) -> Set[str]: - return set(col.column for r in self.ranks.values() for col in r.values()) - - # def apply_scoping(self, collection) -> "ScopedTreeRecord": - # return self - - # def to_json(self) -> Dict: - # result = { - # 'ranks': { - # rank: cols['name'].to_json() if len(cols) == 1 else dict(treeNodeCols={k: v.to_json() for k, v in cols.items()}) - # for rank, cols in self.ranks.items() - # }, - # } - # return { 'treeRecord': result } - - # def unparse(self) -> Dict: - # return { 'baseTableName': self.name, 'uploadble': self.to_json() } class MustMatchTreeRecord(TreeRecord): def apply_scoping(self, collection) -> "ScopedMustMatchTreeRecord": diff --git a/specifyweb/workbench/upload/upload_plan_schema.py b/specifyweb/workbench/upload/upload_plan_schema.py index 8cd9c1fdb7c..35dafdf7954 100644 --- a/specifyweb/workbench/upload/upload_plan_schema.py +++ b/specifyweb/workbench/upload/upload_plan_schema.py @@ -8,7 +8,7 @@ from .upload_table import DeferredScopeUploadTable, UploadTable, OneToOneTable, MustMatchTable from .tomany import ToManyRecord -from .treerecord import TreeRecord, ScopedTreeRecord, MustMatchTreeRecord +from .treerecord import TreeRecord, MustMatchTreeRecord from .uploadable import Uploadable from .column_options import ColumnOptions from .scoping import DEFERRED_SCOPING @@ -231,12 +231,15 @@ } -def parse_plan_with_basetable(collection, to_parse: Dict, extra: Dict = {}) -> Tuple[Table, Uploadable]: +def parse_plan_with_basetable(collection, to_parse: Dict) -> Tuple[Table, Uploadable]: base_table = datamodel.get_table_strict(to_parse['baseTableName']) - return base_table, parse_uploadable(collection, base_table, to_parse['uploadable']) + extra = {} + if 'taxonTreeId' in to_parse.keys(): + extra['treedefid'] = to_parse['taxonTreeId'] + return base_table, parse_uploadable(collection, base_table, to_parse['uploadable'], extra) -def parse_plan(collection, to_parse: Dict, extra: Dict = {}) -> Uploadable: - return parse_plan_with_basetable(collection, to_parse, extra)[1] +def parse_plan(collection, to_parse: Dict) -> Uploadable: + return parse_plan_with_basetable(collection, to_parse)[1] def parse_uploadable(collection, table: Table, to_parse: Dict, extra: Dict = {}) -> Uploadable: if 'uploadTable' in to_parse: @@ -253,8 +256,8 @@ def parse_uploadable(collection, table: Table, to_parse: Dict, extra: Dict = {}) if 'treeRecord' in to_parse: treedefid = None - if 'treedefid' in to_parse.keys(): - treedefid = int(to_parse['treedefid']) + if 'treedefid' in extra.keys(): + treedefid = int(extra['treedefid']) return parse_tree_record(collection, table, to_parse['treeRecord'], treedefid) raise ValueError('unknown uploadable type') @@ -306,7 +309,7 @@ def defer_scope_upload_table(default_collection, table: Table, to_parse: Dict, d } ) -def parse_tree_record(collection, table: Table, to_parse: Dict) -> TreeRecord: +def parse_tree_record(collection, table: Table, to_parse: Dict, base_treedefid: Optional[int] = None) -> TreeRecord: ranks = { rank: {'name': parse_column_options(name_or_cols)} if isinstance(name_or_cols, str) else {k: parse_column_options(v) for k,v in name_or_cols['treeNodeCols'].items() } @@ -315,69 +318,46 @@ def parse_tree_record(collection, table: Table, to_parse: Dict) -> TreeRecord: for rank, cols in ranks.items(): assert 'name' in cols, to_parse + if base_treedefid is None: + for rank, cols in ranks.items(): + treedefid = get_treedef_id(cols['name'].column, False, base_treedefid) + if treedefid is not None: + base_treedefid = treedefid + break + return TreeRecord( name=table.django_name, ranks=ranks, + treedef_id=base_treedefid ) + +def get_treedef_id(rank_name: str, is_adjusting: bool = False, base_treedef_id: Optional[int] = None) -> Optional[int]: + def find_treedef(treedef_id: Optional[int] = None): + filter_kwargs = {'name': rank_name} + if treedef_id is not None: + filter_kwargs['treedef_id'] = str(treedef_id) # or treedef_id ? + return Taxontreedefitem.objects.filter(**filter_kwargs).first() -def parse_tree_record_2(collection, table: Table, to_parse: Dict, base_treedef_id: Optional[int] = None) -> TreeRecord: - ranks = {} - ranks_treedefid_map = {} - treedefitems = [] - for rank, name_or_cols in to_parse['ranks'].items(): - if isinstance(name_or_cols, str): - ranks[rank] = {'name': parse_column_options(name_or_cols)} - else: - ranks[rank] = {} - for k, v in name_or_cols['treeNodeCols'].items(): - ranks[rank][k] = parse_column_options(v) - if 'treedefid' in name_or_cols.keys(): - treedef_id = int(name_or_cols['treedefid']) - ranks_treedefid_map[rank] = treedef_id - treedefitems.append(Taxontreedefitem.objects.filter(treedef_id=treedef_id, name=rank).first()) - ranks[rank]['treedefid'] = treedef_id - elif 'treedefname' in name_or_cols.keys(): - treedef_id = Taxontreedef.objects.filter(name=name_or_cols['treedefname']).first().id - ranks[rank]['treedefid'] = str(treedef_id) - else: - taxontreedefitems = Taxontreedefitem.objects.filter(name=rank) - # if taxontreedefitems.count() > 1: - # raise ValueError(f"Multiple treedefitems with name {rank}") - treedefitem = taxontreedefitems.first() - # if treedefitem is None: - # raise ValueError(f"Could not find treedefitem with name {rank}") - treedef_id = treedefitem.treedef_id - ranks[rank]['treedefid'] = str(treedef_id) - # TODO: Handle cases where two ranks have the same name but different treedefids + treedef = find_treedef(base_treedef_id) + if treedef: + return treedef.id - for rank, cols in ranks.items(): - assert 'name' in cols, to_parse + treedef = find_treedef() + if treedef: + return treedef.id - return TreeRecord( - name=table.django_name, - ranks=ranks - ) - -def adjust_upload_plan(plan: Dict) -> Dict: - def get_treedef_id(rank_name: str, base_treedef_id: Optional[int]) -> int: - if base_treedef_id is not None: - treedef = Taxontreedefitem.objects.filter(treedef_id=base_treedef_id, name=rank_name).first() - if treedef: - if treedef.count() > 1: - raise ValueError(f"Multiple treedefitems with name {rank_name} and treedef_id {base_treedef_id}") - return base_treedef_id - treedef = Taxontreedefitem.objects.filter(name=rank_name).first() - if treedef: - return treedef.id - raise DoesNotExistError(f"Could not find treedefitem with name {rank_name}") + if is_adjusting: + raise ValueError(f"Could not find treedefitem with name {rank_name}") + return None +def adjust_upload_plan(plan: Dict) -> Dict: if plan['baseTableName'] == 'taxon': base_treedef_id = plan.get('taxonTreeId', None) for rank, name_or_cols in plan['uploadable']['treeRecord']['ranks'].items(): if isinstance(name_or_cols, dict): rank_name = name_or_cols['treeNodeCols']['name'] if 'treedefid' not in name_or_cols: - name_or_cols['treedefid'] = get_treedef_id(rank_name, base_treedef_id) + name_or_cols['treedefid'] = str(get_treedef_id(rank_name, True, base_treedef_id)) return plan diff --git a/specifyweb/workbench/views.py b/specifyweb/workbench/views.py index f3a271bd59a..70bb7270c9c 100644 --- a/specifyweb/workbench/views.py +++ b/specifyweb/workbench/views.py @@ -480,15 +480,10 @@ def dataset(request, ds: models.Spdataset) -> http.HttpResponse: # TODO fix this # return http.HttpResponse(f"upload plan is invalid: {e}", status=400) pass - - extra = {} - if 'taxonTreeId' in plan.keys(): - extra['treedefid'] = plan['taxonTreeId'] - parsed_plan = upload_plan_schema.parse_plan(request.specify_collection, plan, extra) + parsed_plan = upload_plan_schema.parse_plan(request.specify_collection, plan) new_cols = parsed_plan.get_cols() - set(ds.columns) if plan['baseTableName'] == 'taxon': - # plan['uploadable'] = parsed_plan.to_json() plan = upload_plan_schema.adjust_upload_plan(plan) if new_cols: ncols = len(ds.columns) From 57f9a09f7d538173e366b004c92c7368866639fb Mon Sep 17 00:00:00 2001 From: alec_dev Date: Tue, 20 Aug 2024 12:19:09 -0500 Subject: [PATCH 18/78] fix taxon treedef scoping logic --- specifyweb/workbench/upload/scoping.py | 3 ++- specifyweb/workbench/upload/upload_plan_schema.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/specifyweb/workbench/upload/scoping.py b/specifyweb/workbench/upload/scoping.py index 7815fba9489..a36b65f47ae 100644 --- a/specifyweb/workbench/upload/scoping.py +++ b/specifyweb/workbench/upload/scoping.py @@ -186,10 +186,11 @@ def apply_scoping_to_tomanyrecord(tmr: ToManyRecord, collection) -> ScopedToMany def apply_scoping_to_treerecord(tr: TreeRecord, collection) -> ScopedTreeRecord: table = datamodel.get_table_strict(tr.name) + treedef = None if table.name == 'Taxon': if tr.treedef_id is not None: treedef = models.Taxontreedef.objects.filter(id=tr.treedef_id).first() - else: + if treedef is None: treedef = collection.discipline.taxontreedef elif table.name == 'Geography': diff --git a/specifyweb/workbench/upload/upload_plan_schema.py b/specifyweb/workbench/upload/upload_plan_schema.py index 35dafdf7954..63a31f0127a 100644 --- a/specifyweb/workbench/upload/upload_plan_schema.py +++ b/specifyweb/workbench/upload/upload_plan_schema.py @@ -331,7 +331,7 @@ def parse_tree_record(collection, table: Table, to_parse: Dict, base_treedefid: treedef_id=base_treedefid ) -def get_treedef_id(rank_name: str, is_adjusting: bool = False, base_treedef_id: Optional[int] = None) -> Optional[int]: +def get_treedef_id(rank_name: str, is_adjusting: bool, base_treedef_id: Optional[int] = None) -> Optional[int]: def find_treedef(treedef_id: Optional[int] = None): filter_kwargs = {'name': rank_name} if treedef_id is not None: From c007b2916ce550a7a9c4772a30eb4a2d617ba54f Mon Sep 17 00:00:00 2001 From: alec_dev Date: Tue, 20 Aug 2024 12:55:06 -0500 Subject: [PATCH 19/78] notes --- specifyweb/workbench/upload/treerecord.py | 5 ++--- specifyweb/workbench/upload/upload_plan_schema.py | 3 ++- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/specifyweb/workbench/upload/treerecord.py b/specifyweb/workbench/upload/treerecord.py index 27493fc9d27..e6cd9e43281 100644 --- a/specifyweb/workbench/upload/treerecord.py +++ b/specifyweb/workbench/upload/treerecord.py @@ -23,7 +23,7 @@ class TreeRecord(NamedTuple): name: str ranks: Dict[str, Dict[str, ColumnOptions]] - treedef_id: Optional[int] = None + treedef_id: Optional[int] = None # TODO: Determine if this is needed def apply_scoping(self, collection) -> "ScopedTreeRecord": from .scoping import apply_scoping_to_treerecord as apply_scoping @@ -51,8 +51,7 @@ def to_json(self) -> Dict: return {'treeRecord': result} def unparse(self) -> Dict: - return { 'baseTableName': self.name, - 'uploadble': self.to_json() } + return { 'baseTableName': self.name, 'uploadble': self.to_json() } class ScopedTreeRecord(NamedTuple): name: str diff --git a/specifyweb/workbench/upload/upload_plan_schema.py b/specifyweb/workbench/upload/upload_plan_schema.py index 63a31f0127a..e18e8b980e6 100644 --- a/specifyweb/workbench/upload/upload_plan_schema.py +++ b/specifyweb/workbench/upload/upload_plan_schema.py @@ -309,6 +309,7 @@ def defer_scope_upload_table(default_collection, table: Table, to_parse: Dict, d } ) +# TODO: Determine if it is better to return ScopedTreeRecord def parse_tree_record(collection, table: Table, to_parse: Dict, base_treedefid: Optional[int] = None) -> TreeRecord: ranks = { rank: {'name': parse_column_options(name_or_cols)} if isinstance(name_or_cols, str) @@ -335,7 +336,7 @@ def get_treedef_id(rank_name: str, is_adjusting: bool, base_treedef_id: Optional def find_treedef(treedef_id: Optional[int] = None): filter_kwargs = {'name': rank_name} if treedef_id is not None: - filter_kwargs['treedef_id'] = str(treedef_id) # or treedef_id ? + filter_kwargs['treedef_id'] = str(treedef_id) return Taxontreedefitem.objects.filter(**filter_kwargs).first() treedef = find_treedef(base_treedef_id) From 9094d6b4c657e39e093ed41c4c1cdf3bfec255e6 Mon Sep 17 00:00:00 2001 From: alec_dev Date: Tue, 20 Aug 2024 15:08:45 -0500 Subject: [PATCH 20/78] fix find_treedef --- .../workbench/upload/upload_plan_schema.py | 36 +++++++++++++++++-- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/specifyweb/workbench/upload/upload_plan_schema.py b/specifyweb/workbench/upload/upload_plan_schema.py index e18e8b980e6..814f87fbdf9 100644 --- a/specifyweb/workbench/upload/upload_plan_schema.py +++ b/specifyweb/workbench/upload/upload_plan_schema.py @@ -331,13 +331,19 @@ def parse_tree_record(collection, table: Table, to_parse: Dict, base_treedefid: ranks=ranks, treedef_id=base_treedefid ) - + def get_treedef_id(rank_name: str, is_adjusting: bool, base_treedef_id: Optional[int] = None) -> Optional[int]: - def find_treedef(treedef_id: Optional[int] = None): + def find_treedef(treedef_id: Optional[int] = None, is_adjusting: bool = False) -> Optional['Taxontreedef']: filter_kwargs = {'name': rank_name} if treedef_id is not None: filter_kwargs['treedef_id'] = str(treedef_id) - return Taxontreedefitem.objects.filter(**filter_kwargs).first() + ranks = Taxontreedefitem.objects.filter(**filter_kwargs) + if ranks.count() == 0: + return None + elif ranks.count() > 1: + if is_adjusting: + raise ValueError(f"Multiple treedefitems with name {rank_name}") + return ranks.first().treedef treedef = find_treedef(base_treedef_id) if treedef: @@ -351,6 +357,30 @@ def find_treedef(treedef_id: Optional[int] = None): raise ValueError(f"Could not find treedefitem with name {rank_name}") return None + +def find_treedef( + rank_name: str, treedef_id: Optional[int] = None, is_adjusting: bool = False +) -> Optional["Taxontreedef"]: + filter_kwargs = {'name': rank_name} + if treedef_id is not None: + filter_kwargs['treedef_id'] = str(treedef_id) + ranks = Taxontreedefitem.objects.filter(**filter_kwargs) + if ranks.count() == 0: + return None + elif ranks.count() > 1 and is_adjusting and treedef_id is not None: + raise ValueError(f"Multiple treedefitems with name {rank_name} and treedef_id {treedef_id}") + return ranks.first().treedef + +def get_treedef_id(rank_name: str, is_adjusting: bool, base_treedef_id: Optional[int] = None) -> Optional[int]: + for treedef_id in [base_treedef_id, None]: + treedef = find_treedef(rank_name, treedef_id, is_adjusting) + if treedef: + return treedef.id + + if is_adjusting: + raise ValueError(f"Could not find treedefitem with name {rank_name}") + return None + def adjust_upload_plan(plan: Dict) -> Dict: if plan['baseTableName'] == 'taxon': base_treedef_id = plan.get('taxonTreeId', None) From 7c052d2bb3f2e133ee85dc462b734f84546caa6b Mon Sep 17 00:00:00 2001 From: alec_dev Date: Tue, 20 Aug 2024 15:45:21 -0500 Subject: [PATCH 21/78] fix mypy error --- .../workbench/upload/upload_plan_schema.py | 32 ++++--------------- 1 file changed, 6 insertions(+), 26 deletions(-) diff --git a/specifyweb/workbench/upload/upload_plan_schema.py b/specifyweb/workbench/upload/upload_plan_schema.py index 814f87fbdf9..d754cc813ca 100644 --- a/specifyweb/workbench/upload/upload_plan_schema.py +++ b/specifyweb/workbench/upload/upload_plan_schema.py @@ -332,31 +332,6 @@ def parse_tree_record(collection, table: Table, to_parse: Dict, base_treedefid: treedef_id=base_treedefid ) -def get_treedef_id(rank_name: str, is_adjusting: bool, base_treedef_id: Optional[int] = None) -> Optional[int]: - def find_treedef(treedef_id: Optional[int] = None, is_adjusting: bool = False) -> Optional['Taxontreedef']: - filter_kwargs = {'name': rank_name} - if treedef_id is not None: - filter_kwargs['treedef_id'] = str(treedef_id) - ranks = Taxontreedefitem.objects.filter(**filter_kwargs) - if ranks.count() == 0: - return None - elif ranks.count() > 1: - if is_adjusting: - raise ValueError(f"Multiple treedefitems with name {rank_name}") - return ranks.first().treedef - - treedef = find_treedef(base_treedef_id) - if treedef: - return treedef.id - - treedef = find_treedef() - if treedef: - return treedef.id - - if is_adjusting: - raise ValueError(f"Could not find treedefitem with name {rank_name}") - return None - def find_treedef( rank_name: str, treedef_id: Optional[int] = None, is_adjusting: bool = False @@ -369,7 +344,12 @@ def find_treedef( return None elif ranks.count() > 1 and is_adjusting and treedef_id is not None: raise ValueError(f"Multiple treedefitems with name {rank_name} and treedef_id {treedef_id}") - return ranks.first().treedef + + first_rank = ranks.first() + if first_rank is None: + return None + + return first_rank.treedef def get_treedef_id(rank_name: str, is_adjusting: bool, base_treedef_id: Optional[int] = None) -> Optional[int]: for treedef_id in [base_treedef_id, None]: From 6abe879b0c54baed8c8116004f9c2b7e53956848 Mon Sep 17 00:00:00 2001 From: alec_dev Date: Wed, 21 Aug 2024 10:23:00 -0500 Subject: [PATCH 22/78] allow for non-background wb uploads --- specifyweb/workbench/tasks.py | 50 ++++++++++++++++++++++------------- specifyweb/workbench/views.py | 46 +++++++++++++++++++++----------- 2 files changed, 62 insertions(+), 34 deletions(-) diff --git a/specifyweb/workbench/tasks.py b/specifyweb/workbench/tasks.py index ea24552b954..295b75129bc 100644 --- a/specifyweb/workbench/tasks.py +++ b/specifyweb/workbench/tasks.py @@ -1,6 +1,7 @@ from typing import Optional, Any from celery import shared_task # type: ignore +from celery import Task # type: ignore from celery.utils.log import get_task_logger # type: ignore from django.db import connection, transaction @@ -14,37 +15,50 @@ logger = get_task_logger(__name__) -@app.task(base=LogErrorsTask, bind=True) -def upload(self, collection_id: int, uploading_agent_id: int, ds_id: int, no_commit: bool, allow_partial: bool) -> None: - - def progress(current: int, total: Optional[int]) -> None: - if not self.request.called_directly: - self.update_state(state='PROGRESS', meta={'current': current, 'total': total}) +def upload_data( + collection_id: int, + uploading_agent_id: int, + ds_id: int, + no_commit: bool, + allow_partial: bool, + task: Optional[Task]=None, + progress=None, +) -> None: with transaction.atomic(): ds = Spdataset.objects.select_for_update().get(id=ds_id) collection = Collection.objects.get(id=collection_id) - if ds.uploaderstatus is None: - logger.info("dataset is not assigned to an upload task") - return + if task is not None: + if ds.uploaderstatus is None: + logger.info("dataset is not assigned to an upload task") + return - if ds.uploaderstatus['taskid'] != self.request.id: - logger.info("dataset is not assigned to this task") - return + if ds.uploaderstatus['taskid'] != task.request.id: + logger.info("dataset is not assigned to this task") + return - if not (ds.uploaderstatus['operation'] == ("validating" if no_commit else "uploading")): raise AssertionError( - f"Invalid status '{ds.uploaderstatus['operation']}' for upload. Expected 'validating' or 'uploading'", - {"uploadStatus" : ds.uploaderstatus['operation'], - "operation" : "upload", - "expectedUploadStatus" : 'validating , uploading', - "localizationKey" : "invalidUploadStatus"}) + if not (ds.uploaderstatus['operation'] == ("validating" if no_commit else "uploading")): raise AssertionError( + f"Invalid status '{ds.uploaderstatus['operation']}' for upload. Expected 'validating' or 'uploading'", + {"uploadStatus" : ds.uploaderstatus['operation'], + "operation" : "upload", + "expectedUploadStatus" : 'validating , uploading', + "localizationKey" : "invalidUploadStatus"}) do_upload_dataset(collection, uploading_agent_id, ds, no_commit, allow_partial, progress) ds.uploaderstatus = None ds.save(update_fields=['uploaderstatus']) +@app.task(base=LogErrorsTask, bind=True) +def upload(self, collection_id: int, uploading_agent_id: int, ds_id: int, no_commit: bool, allow_partial: bool) -> None: + + def progress(current: int, total: Optional[int]) -> None: + if not self.request.called_directly: + self.update_state(state='PROGRESS', meta={'current': current, 'total': total}) + + upload_data(collection_id, uploading_agent_id, ds_id, no_commit, allow_partial, self, progress) + @app.task(base=LogErrorsTask, bind=True) def unupload(self, ds_id: int, agent_id: int) -> None: diff --git a/specifyweb/workbench/views.py b/specifyweb/workbench/views.py index 70bb7270c9c..381df068b5e 100644 --- a/specifyweb/workbench/views.py +++ b/specifyweb/workbench/views.py @@ -1,5 +1,6 @@ import json import logging +import re from typing import List, Optional from uuid import uuid4 @@ -629,22 +630,35 @@ def upload(request, ds, no_commit: bool, allow_partial: bool) -> http.HttpRespon if ds.was_uploaded(): return http.HttpResponse('dataset has already been uploaded.', status=400) - taskid = str(uuid4()) - async_result = tasks.upload.apply_async([ - request.specify_collection.id, - request.specify_user_agent.id, - ds.id, - no_commit, - allow_partial - ], task_id=taskid) - ds.uploaderstatus = { - 'operation': "validating" if no_commit else "uploading", - 'taskid': taskid - } - ds.save(update_fields=['uploaderstatus']) - - return http.JsonResponse(async_result.id, safe=False) - + data = json.loads(request.body) if request.body else {} + background = True + if 'background' in data: + background = bool(data['background']) # {"background": false} + + if background: + taskid = str(uuid4()) + async_result = tasks.upload.apply_async([ + request.specify_collection.id, + request.specify_user_agent.id, + ds.id, + no_commit, + allow_partial + ], task_id=taskid) + ds.uploaderstatus = { + 'operation': "validating" if no_commit else "uploading", + 'taskid': taskid + } + ds.save(update_fields=['uploaderstatus']) + return http.JsonResponse(async_result.id, safe=False) + else: + tasks.upload_data( + request.specify_collection.id, + request.specify_user_agent.id, + ds.id, + no_commit, + allow_partial, + ) + return http.JsonResponse(None, safe=False) @openapi(schema={ 'post': { From 65c364d145744b7c8d4c660ef44bf8cafa2f0393 Mon Sep 17 00:00:00 2001 From: alec_dev Date: Thu, 22 Aug 2024 11:45:07 -0500 Subject: [PATCH 23/78] change TreeRecord ranks key type to include treedefid --- specifyweb/workbench/upload/scoping.py | 44 ++++- specifyweb/workbench/upload/treerecord.py | 159 +++++++++++++++++- .../workbench/upload/upload_plan_schema.py | 59 ++++++- 3 files changed, 244 insertions(+), 18 deletions(-) diff --git a/specifyweb/workbench/upload/scoping.py b/specifyweb/workbench/upload/scoping.py index a36b65f47ae..2efcf38fcff 100644 --- a/specifyweb/workbench/upload/scoping.py +++ b/specifyweb/workbench/upload/scoping.py @@ -9,7 +9,7 @@ from .uploadable import Uploadable, ScopedUploadable from .upload_table import UploadTable, DeferredScopeUploadTable, ScopedUploadTable, OneToOneTable, ScopedOneToOneTable from .tomany import ToManyRecord, ScopedToManyRecord -from .treerecord import TreeRecord, ScopedTreeRecord +from .treerecord import TreeRank, TreeRankRecord, TreeRecord, ScopedTreeRecord from .column_options import ColumnOptions, ExtendedColumnOptions """ There are cases in which the scoping of records should be dependent on another record/column in a WorkBench dataset. @@ -188,8 +188,8 @@ def apply_scoping_to_treerecord(tr: TreeRecord, collection) -> ScopedTreeRecord: treedef = None if table.name == 'Taxon': - if tr.treedef_id is not None: - treedef = models.Taxontreedef.objects.filter(id=tr.treedef_id).first() + if tr.base_treedef_id is not None: + treedef = models.Taxontreedef.objects.filter(id=tr.base_treedef_id).first() if treedef is None: treedef = collection.discipline.taxontreedef @@ -214,16 +214,50 @@ def apply_scoping_to_treerecord(tr: TreeRecord, collection) -> ScopedTreeRecord: treedefitems = list(treedef.treedefitems.order_by('rankid')) treedef_ranks = [tdi.name for tdi in treedefitems] for rank in tr.ranks: - if rank not in treedef_ranks: + if rank not in treedef_ranks and isinstance(rank, TreeRankRecord) and not rank.check_rank(table.name): raise Exception(f'"{rank}" not among {table.name} tree ranks: {treedef_ranks}') root = list(getattr(models, table.name.capitalize()).objects.filter(definitionitem=treedefitems[0])[:1]) # assume there is only one + scoped_ranks: Dict[TreeRankRecord, Dict[str, ExtendedColumnOptions]] = {} + for r, cols in tr.ranks.items(): + if isinstance(r, str): + r = TreeRank(r, table.name, treedef.id if treedef else None, tr.base_treedef_id).tree_rank_record() + scoped_ranks[r] = {} + for f, colopts in cols.items(): + scoped_ranks[r][f] = extend_columnoptions(colopts, collection, table.name, f) + return ScopedTreeRecord( name=tr.name, - ranks={r: {f: extend_columnoptions(colopts, collection, table.name, f) for f, colopts in cols.items()} for r, cols in tr.ranks.items()}, + # ranks={ + # r: { + # f: extend_columnoptions(colopts, collection, table.name, f) + # for f, colopts in cols.items() + # } + # for r, cols in tr.ranks.items() + # }, + ranks=scoped_ranks, treedef=treedef, treedefitems=treedefitems, root=root[0] if root else None, disambiguation={}, ) + +# def scope_taxon_rank(rank_name: str, treedef_id: Optional[int] = None) -> Optional[int]: +# # rank_id = None +# treedefitems = None +# if treedef_id is None: +# treedefitems = models.Taxontreedefitem.objects.filter(name=rank_name, treedef_id=treedef_id) +# else: +# treedefitems = models.Taxontreedefitem.objects.filter(name=rank_name) + +# if treedefitems is None: +# return None + +# if treedefitems.count() == 1: +# # rank_id = treedefitems.first().id +# return treedefitems.first().id +# elif treedefitems.count() > 1: +# raise Exception(f"Multiple treedefitems found for rank {rank_name}") +# elif treedefitems.count() == 0: +# return None diff --git a/specifyweb/workbench/upload/treerecord.py b/specifyweb/workbench/upload/treerecord.py index e6cd9e43281..b0b4ce12ff5 100644 --- a/specifyweb/workbench/upload/treerecord.py +++ b/specifyweb/workbench/upload/treerecord.py @@ -19,11 +19,156 @@ logger = logging.getLogger(__name__) +class TreeRank: + rank_name: str + treedef_id: int + tree: str + + def __init__( + self, + rank_name: str, + tree: str, + treedef_id: Optional[int] = None, + base_treedef_id: Optional[int] = None, + ): + self.rank_name = rank_name + self.treedef_id = self._get_treedef_id(rank_name, tree, treedef_id, base_treedef_id) + self.tree = tree.lower() + + def _get_treedef_id( + self, + rank_name: str, + tree: str, + treedef_id: Optional[int], + base_treedef_id: Optional[int], + ) -> int: + filter_kwargs = self._build_filter_kwargs(rank_name, treedef_id) + tree_model = getattr(models, tree.lower().title() + 'treedefitem') + treedefitems = tree_model.objects.filter(**filter_kwargs) + + if not treedefitems.exists(): + raise ValueError(f"No treedefitems found for rank {rank_name}") + + if treedefitems.count() > 1: + treedefitems = self._filter_by_base_treedef_id(treedefitems, rank_name, base_treedef_id) + + first_item = treedefitems.first() + if first_item is None: + raise ValueError(f"No treedefitems found for rank {rank_name}") + + return first_item.treedef_id + + def _build_filter_kwargs(self, rank_name: str, treedef_id: Optional[int]) -> Dict[str, Any]: + filter_kwargs = {'name': rank_name} + if treedef_id is not None: + filter_kwargs['treedef_id'] = treedef_id # type: ignore + return filter_kwargs + + def _filter_by_base_treedef_id(self, treedefitems, rank_name: str, base_treedef_id: Optional[int]): + if base_treedef_id is None: + raise ValueError(f"Multiple treedefitems found for rank {rank_name}") + treedefitems = treedefitems.filter(treedef_id=base_treedef_id) + if not treedefitems.exists(): + raise ValueError(f"No treedefitems found for rank {rank_name} and base treedef {base_treedef_id}") + if treedefitems.count() > 1: + raise ValueError(f"Multiple treedefitems found for rank {rank_name} and base treedef {base_treedef_id}") + return treedefitems + + # def get_tree_model(self): + # return getattr(models, self.tree.lower().title() + 'treedefitem') + + def check_rank(self) -> bool: + tree_model = getattr(models, self.tree.lower().title() + 'treedefitem') + rank = tree_model.objects.filter(name=self.rank_name, treedef_id=self.treedef_id) + return rank.exists() and rank.count() == 1 + + def validate_rank(self) -> None: + if not self.check_rank(): + raise ValueError(f"Invalid rank {self.rank_name} for treedef {self.treedef_id}") + + def tree_rank_record(self) -> 'TreeRankRecord': + assert self.rank_name is not None, "Rank name is required" + assert self.treedef_id is not None, "Treedef ID is required" + assert self.tree is not None and self.tree.lower() in { + "taxon", "storage", "geography", "geologictimeperiod", "lithostrat" # TODO: Replace with constants + }, "Tree is required" + return TreeRankRecord(self.rank_name, self.treedef_id) + +class TreeRankRecord(NamedTuple): + rank_name: str + treedef_id: int + + def to_json(self) -> Dict: + return { + "rank": self.rank_name, + "treedefId": self.treedef_id, + } + + def check_rank(self, tree: str) -> bool: + return TreeRank(self.rank_name, tree, self.treedef_id).check_rank() + + def validate_rank(self, tree) -> None: + TreeRank(self.rank_name, tree, self.treedef_id).validate_rank() + +# def create_tree_record(rank_name: str, tree: str, treedef_id: Optional[int] = None, base_treedef_id: Optional[int] = None) -> TreeRankRecord: +# # tree_id = +# pass + +# def get_treedef_id(rank_name: str, tree: str, treedef_id: Optional[int], base_treedef_id: Optional[int]) -> int: +# filter_kwargs = build_filter_kwargs(rank_name, treedef_id) +# tree_model = getattr(models, tree.lower().title() + 'treedefitem') +# treedefitems = tree_model.objects.filter(**filter_kwargs) + +# if not treedefitems.exists(): +# raise ValueError(f"No treedefitems found for rank {rank_name}") + +# if treedefitems.count() > 1: +# treedefitems = filter_by_base_treedef_id(treedefitems, rank_name, base_treedef_id) + +# first_item = treedefitems.first() +# if first_item is None: +# raise ValueError(f"No treedefitems found for rank {rank_name}") + +# return first_item.treedef_id + +# def build_filter_kwargs(rank_name: str, treedef_id: Optional[int]) -> Dict[str, Any]: +# filter_kwargs = {'name': rank_name} +# if treedef_id is not None: +# filter_kwargs['treedef_id'] = treedef_id # type: ignore +# return filter_kwargs + +# def filter_by_base_treedef_id(self, treedefitems, rank_name: str, base_treedef_id: Optional[int]): +# if base_treedef_id is None: +# raise ValueError(f"Multiple treedefitems found for rank {rank_name}") +# treedefitems = treedefitems.filter(treedef_id=base_treedef_id) +# if not treedefitems.exists(): +# raise ValueError(f"No treedefitems found for rank {rank_name} and base treedef {base_treedef_id}") +# if treedefitems.count() > 1: +# raise ValueError(f"Multiple treedefitems found for rank {rank_name} and base treedef {base_treedef_id}") +# return treedefitems + +# def check_rank() -> bool: +# rank = models.Taxontreedefitem.objects.filter(name=rank_name, treedef_id=treedef_id) +# return rank.exists() and rank.count() == 1 + +# def validate_rank() -> None: +# if not self.check_rank(): +# raise ValueError(f"Invalid rank {self.rank_name} for treedef {self.treedef_id}") + +# def tree_rank_record() -> 'TreeRankRecord': +# assert self.rank_name is not None, "Rank name is required" +# assert self.treedef_id is not None, "Treedef ID is required" +# assert self.tree is not None and self.tree.lower() in { +# "taxon", "storage", "geography", "geologictimeperiod", "lithostrat" # TODO: Replace with constants +# }, "Tree is required" +# return TreeRankRecord(self.rank_name, self.treedef_id) class TreeRecord(NamedTuple): name: str - ranks: Dict[str, Dict[str, ColumnOptions]] - treedef_id: Optional[int] = None # TODO: Determine if this is needed + # ranks: Dict[str, Dict[str, ColumnOptions]] + # ranks: Dict[TreeRankRecord, Dict[str, ColumnOptions]] + ranks: Dict[Union[str, TreeRankRecord], Dict[str, ColumnOptions]] + base_treedef_id: Optional[int] = None def apply_scoping(self, collection) -> "ScopedTreeRecord": from .scoping import apply_scoping_to_treerecord as apply_scoping @@ -55,7 +200,7 @@ def unparse(self) -> Dict: class ScopedTreeRecord(NamedTuple): name: str - ranks: Dict[str, Dict[str, ExtendedColumnOptions]] + ranks: Dict[TreeRankRecord, Dict[str, ExtendedColumnOptions]] treedef: Any treedefitems: List root: Optional[Any] @@ -70,15 +215,15 @@ def get_treedefs(self) -> Set: def bind(self, collection, row: Row, uploadingAgentId: Optional[int], auditor: Auditor, cache: Optional[Dict]=None, row_index: Optional[int] = None) -> Union["BoundTreeRecord", ParseFailures]: parsedFields: Dict[str, List[ParseResult]] = {} parseFails: List[WorkBenchParseFailure] = [] - for rank, cols in self.ranks.items(): + for tree_rank_record, cols in self.ranks.items(): nameColumn = cols['name'] - presults, pfails = parse_many(collection, self.name, cols, row) - parsedFields[rank] = presults + presults, pfails = parse_many(collection, self.name, cols, row) # TODO: fix + parsedFields[tree_rank_record.rank_name] = presults parseFails += pfails filters = {k: v for result in presults for k, v in result.filter_on.items()} if filters.get('name', None) is None: parseFails += [ - WorkBenchParseFailure('invalidPartialRecord',{'column':nameColumn.column}, result.column) + WorkBenchParseFailure('invalidPartialRecord', {'column': nameColumn.column}, result.column) for result in presults if any(v is not None for v in result.filter_on.values()) ] diff --git a/specifyweb/workbench/upload/upload_plan_schema.py b/specifyweb/workbench/upload/upload_plan_schema.py index d754cc813ca..198c2c49ff3 100644 --- a/specifyweb/workbench/upload/upload_plan_schema.py +++ b/specifyweb/workbench/upload/upload_plan_schema.py @@ -8,7 +8,7 @@ from .upload_table import DeferredScopeUploadTable, UploadTable, OneToOneTable, MustMatchTable from .tomany import ToManyRecord -from .treerecord import TreeRecord, MustMatchTreeRecord +from .treerecord import TreeRank, TreeRankRecord, TreeRecord, MustMatchTreeRecord from .uploadable import Uploadable from .column_options import ColumnOptions from .scoping import DEFERRED_SCOPING @@ -100,6 +100,7 @@ 'type': 'object', 'properties': { 'treeNodeCols': { '$ref': '#/definitions/treeNodeCols' }, + 'taxonTreeId': { '$ref': '#definitions/taxonTreeId'} }, 'required': [ 'treeNodeCols' ], 'additionalProperties': False @@ -309,8 +310,7 @@ def defer_scope_upload_table(default_collection, table: Table, to_parse: Dict, d } ) -# TODO: Determine if it is better to return ScopedTreeRecord -def parse_tree_record(collection, table: Table, to_parse: Dict, base_treedefid: Optional[int] = None) -> TreeRecord: +def parse_tree_record_old(collection, table: Table, to_parse: Dict, base_treedefid: Optional[int] = None) -> TreeRecord: ranks = { rank: {'name': parse_column_options(name_or_cols)} if isinstance(name_or_cols, str) else {k: parse_column_options(v) for k,v in name_or_cols['treeNodeCols'].items() } @@ -329,16 +329,62 @@ def parse_tree_record(collection, table: Table, to_parse: Dict, base_treedefid: return TreeRecord( name=table.django_name, ranks=ranks, - treedef_id=base_treedefid + base_treedef_id=base_treedefid ) +def parse_tree_record(collection, table: Table, to_parse: Dict, base_treedefid: Optional[int] = None) -> TreeRecord: + ranks: Dict[Union[str, TreeRankRecord], Dict[str, ColumnOptions]] = {} + for rank, name_or_cols in to_parse['ranks'].items(): + if isinstance(name_or_cols, str): + rank_name = name_or_cols + parsed_cols = {'name': parse_column_options(name_or_cols)} + else: + tree_node_cols = {k: parse_column_options(v) for k, v in name_or_cols['treeNodeCols'].items()} + rank_name = name_or_cols['treeNodeCols']['name'] + parsed_cols = tree_node_cols + + treedefid = get_treedef_id(rank_name, False, base_treedefid) + tree_rank_record = TreeRank(rank, table.name, treedefid, base_treedefid).tree_rank_record() + ranks[tree_rank_record] = parsed_cols + + # for rank, name_or_cols in to_parse['ranks'].items(): + # if isinstance(name_or_cols, str): + # rank_name = name_or_cols + # treedefid = get_treedef_id(rank_name, False, base_treedefid) + # tree_rank_record = TreeRank(rank, table.name, treedefid, base_treedefid).tree_rank_record() + # ranks[tree_rank_record] = {'name': parse_column_options(name_or_cols)} + # else: + # tree_node_cols = {} + # for k, v in name_or_cols['treeNodeCols'].items(): + # tree_node_cols[k] = parse_column_options(v) + + # rank_name = name_or_cols['treeNodeCols']['name'] + # treedefid = get_treedef_id(rank_name, False, base_treedefid) + # tree_rank_record = TreeRank(rank, table.name, treedefid, base_treedefid).tree_rank_record() + # ranks[tree_rank_record] = tree_node_cols + + for rank, cols in ranks.items(): + assert 'name' in cols, to_parse + + if base_treedefid is None: + for tree_rank_record, cols in ranks.items(): # type: ignore + treedefid = get_treedef_id(cols['name'].column, False, base_treedefid) + if treedefid is not None: + base_treedefid = treedefid + break + + return TreeRecord( + name=table.django_name, + ranks=ranks, + base_treedef_id=base_treedefid + ) def find_treedef( rank_name: str, treedef_id: Optional[int] = None, is_adjusting: bool = False ) -> Optional["Taxontreedef"]: filter_kwargs = {'name': rank_name} if treedef_id is not None: - filter_kwargs['treedef_id'] = str(treedef_id) + filter_kwargs['treedef_id'] = treedef_id # type: ignore ranks = Taxontreedefitem.objects.filter(**filter_kwargs) if ranks.count() == 0: return None @@ -368,7 +414,8 @@ def adjust_upload_plan(plan: Dict) -> Dict: if isinstance(name_or_cols, dict): rank_name = name_or_cols['treeNodeCols']['name'] if 'treedefid' not in name_or_cols: - name_or_cols['treedefid'] = str(get_treedef_id(rank_name, True, base_treedef_id)) + # name_or_cols['treedefid'] = str(get_treedef_id(rank_name, True, base_treedef_id)) + name_or_cols['taxonTreeId'] = get_treedef_id(rank_name, True, base_treedef_id) return plan From 7552b87310091b849bafcae7a37ab3f63e916d48 Mon Sep 17 00:00:00 2001 From: alec_dev Date: Thu, 22 Aug 2024 12:06:07 -0500 Subject: [PATCH 24/78] clearnup and fix unparsing unit test --- specifyweb/workbench/upload/scoping.py | 26 -------- specifyweb/workbench/upload/treerecord.py | 63 ++----------------- .../workbench/upload/upload_plan_schema.py | 38 ----------- 3 files changed, 5 insertions(+), 122 deletions(-) diff --git a/specifyweb/workbench/upload/scoping.py b/specifyweb/workbench/upload/scoping.py index 2efcf38fcff..848977c3b9a 100644 --- a/specifyweb/workbench/upload/scoping.py +++ b/specifyweb/workbench/upload/scoping.py @@ -229,35 +229,9 @@ def apply_scoping_to_treerecord(tr: TreeRecord, collection) -> ScopedTreeRecord: return ScopedTreeRecord( name=tr.name, - # ranks={ - # r: { - # f: extend_columnoptions(colopts, collection, table.name, f) - # for f, colopts in cols.items() - # } - # for r, cols in tr.ranks.items() - # }, ranks=scoped_ranks, treedef=treedef, treedefitems=treedefitems, root=root[0] if root else None, disambiguation={}, ) - -# def scope_taxon_rank(rank_name: str, treedef_id: Optional[int] = None) -> Optional[int]: -# # rank_id = None -# treedefitems = None -# if treedef_id is None: -# treedefitems = models.Taxontreedefitem.objects.filter(name=rank_name, treedef_id=treedef_id) -# else: -# treedefitems = models.Taxontreedefitem.objects.filter(name=rank_name) - -# if treedefitems is None: -# return None - -# if treedefitems.count() == 1: -# # rank_id = treedefitems.first().id -# return treedefitems.first().id -# elif treedefitems.count() > 1: -# raise Exception(f"Multiple treedefitems found for rank {rank_name}") -# elif treedefitems.count() == 0: -# return None diff --git a/specifyweb/workbench/upload/treerecord.py b/specifyweb/workbench/upload/treerecord.py index b0b4ce12ff5..55f77413e18 100644 --- a/specifyweb/workbench/upload/treerecord.py +++ b/specifyweb/workbench/upload/treerecord.py @@ -73,9 +73,6 @@ def _filter_by_base_treedef_id(self, treedefitems, rank_name: str, base_treedef_ if treedefitems.count() > 1: raise ValueError(f"Multiple treedefitems found for rank {rank_name} and base treedef {base_treedef_id}") return treedefitems - - # def get_tree_model(self): - # return getattr(models, self.tree.lower().title() + 'treedefitem') def check_rank(self) -> bool: tree_model = getattr(models, self.tree.lower().title() + 'treedefitem') @@ -104,69 +101,19 @@ def to_json(self) -> Dict: "treedefId": self.treedef_id, } + def to_key(self) -> Tuple[str, int]: + return (self.rank_name, self.treedef_id) + def check_rank(self, tree: str) -> bool: return TreeRank(self.rank_name, tree, self.treedef_id).check_rank() def validate_rank(self, tree) -> None: TreeRank(self.rank_name, tree, self.treedef_id).validate_rank() -# def create_tree_record(rank_name: str, tree: str, treedef_id: Optional[int] = None, base_treedef_id: Optional[int] = None) -> TreeRankRecord: -# # tree_id = -# pass - -# def get_treedef_id(rank_name: str, tree: str, treedef_id: Optional[int], base_treedef_id: Optional[int]) -> int: -# filter_kwargs = build_filter_kwargs(rank_name, treedef_id) -# tree_model = getattr(models, tree.lower().title() + 'treedefitem') -# treedefitems = tree_model.objects.filter(**filter_kwargs) - -# if not treedefitems.exists(): -# raise ValueError(f"No treedefitems found for rank {rank_name}") - -# if treedefitems.count() > 1: -# treedefitems = filter_by_base_treedef_id(treedefitems, rank_name, base_treedef_id) - -# first_item = treedefitems.first() -# if first_item is None: -# raise ValueError(f"No treedefitems found for rank {rank_name}") - -# return first_item.treedef_id - -# def build_filter_kwargs(rank_name: str, treedef_id: Optional[int]) -> Dict[str, Any]: -# filter_kwargs = {'name': rank_name} -# if treedef_id is not None: -# filter_kwargs['treedef_id'] = treedef_id # type: ignore -# return filter_kwargs - -# def filter_by_base_treedef_id(self, treedefitems, rank_name: str, base_treedef_id: Optional[int]): -# if base_treedef_id is None: -# raise ValueError(f"Multiple treedefitems found for rank {rank_name}") -# treedefitems = treedefitems.filter(treedef_id=base_treedef_id) -# if not treedefitems.exists(): -# raise ValueError(f"No treedefitems found for rank {rank_name} and base treedef {base_treedef_id}") -# if treedefitems.count() > 1: -# raise ValueError(f"Multiple treedefitems found for rank {rank_name} and base treedef {base_treedef_id}") -# return treedefitems - -# def check_rank() -> bool: -# rank = models.Taxontreedefitem.objects.filter(name=rank_name, treedef_id=treedef_id) -# return rank.exists() and rank.count() == 1 - -# def validate_rank() -> None: -# if not self.check_rank(): -# raise ValueError(f"Invalid rank {self.rank_name} for treedef {self.treedef_id}") - -# def tree_rank_record() -> 'TreeRankRecord': -# assert self.rank_name is not None, "Rank name is required" -# assert self.treedef_id is not None, "Treedef ID is required" -# assert self.tree is not None and self.tree.lower() in { -# "taxon", "storage", "geography", "geologictimeperiod", "lithostrat" # TODO: Replace with constants -# }, "Tree is required" -# return TreeRankRecord(self.rank_name, self.treedef_id) - class TreeRecord(NamedTuple): name: str # ranks: Dict[str, Dict[str, ColumnOptions]] - # ranks: Dict[TreeRankRecord, Dict[str, ColumnOptions]] + # ranks: Dict[Union[str, Tuple[str, int]], Dict[str, ColumnOptions]] ranks: Dict[Union[str, TreeRankRecord], Dict[str, ColumnOptions]] base_treedef_id: Optional[int] = None @@ -180,7 +127,7 @@ def get_cols(self) -> Set[str]: def to_json(self) -> Dict: result = { "ranks": { - rank: ( + rank.rank_name if isinstance(rank, TreeRankRecord) else rank: ( cols["name"].to_json() if len(cols) == 1 else dict( diff --git a/specifyweb/workbench/upload/upload_plan_schema.py b/specifyweb/workbench/upload/upload_plan_schema.py index 198c2c49ff3..7c50ae8f0bb 100644 --- a/specifyweb/workbench/upload/upload_plan_schema.py +++ b/specifyweb/workbench/upload/upload_plan_schema.py @@ -310,28 +310,6 @@ def defer_scope_upload_table(default_collection, table: Table, to_parse: Dict, d } ) -def parse_tree_record_old(collection, table: Table, to_parse: Dict, base_treedefid: Optional[int] = None) -> TreeRecord: - ranks = { - rank: {'name': parse_column_options(name_or_cols)} if isinstance(name_or_cols, str) - else {k: parse_column_options(v) for k,v in name_or_cols['treeNodeCols'].items() } - for rank, name_or_cols in to_parse['ranks'].items() - } - for rank, cols in ranks.items(): - assert 'name' in cols, to_parse - - if base_treedefid is None: - for rank, cols in ranks.items(): - treedefid = get_treedef_id(cols['name'].column, False, base_treedefid) - if treedefid is not None: - base_treedefid = treedefid - break - - return TreeRecord( - name=table.django_name, - ranks=ranks, - base_treedef_id=base_treedefid - ) - def parse_tree_record(collection, table: Table, to_parse: Dict, base_treedefid: Optional[int] = None) -> TreeRecord: ranks: Dict[Union[str, TreeRankRecord], Dict[str, ColumnOptions]] = {} for rank, name_or_cols in to_parse['ranks'].items(): @@ -347,22 +325,6 @@ def parse_tree_record(collection, table: Table, to_parse: Dict, base_treedefid: tree_rank_record = TreeRank(rank, table.name, treedefid, base_treedefid).tree_rank_record() ranks[tree_rank_record] = parsed_cols - # for rank, name_or_cols in to_parse['ranks'].items(): - # if isinstance(name_or_cols, str): - # rank_name = name_or_cols - # treedefid = get_treedef_id(rank_name, False, base_treedefid) - # tree_rank_record = TreeRank(rank, table.name, treedefid, base_treedefid).tree_rank_record() - # ranks[tree_rank_record] = {'name': parse_column_options(name_or_cols)} - # else: - # tree_node_cols = {} - # for k, v in name_or_cols['treeNodeCols'].items(): - # tree_node_cols[k] = parse_column_options(v) - - # rank_name = name_or_cols['treeNodeCols']['name'] - # treedefid = get_treedef_id(rank_name, False, base_treedefid) - # tree_rank_record = TreeRank(rank, table.name, treedefid, base_treedefid).tree_rank_record() - # ranks[tree_rank_record] = tree_node_cols - for rank, cols in ranks.items(): assert 'name' in cols, to_parse From 4eb6730cdcbd1da3aaca87992cdaf10aec861b37 Mon Sep 17 00:00:00 2001 From: alec_dev Date: Fri, 23 Aug 2024 14:24:10 -0500 Subject: [PATCH 25/78] reformulate and change taxonTreeId to treeId in schema --- .../workbench/upload/tests/example_plan.py | 2 + specifyweb/workbench/upload/treerecord.py | 46 +++-- .../workbench/upload/upload_plan_schema.py | 177 ++++++++++++------ specifyweb/workbench/views.py | 4 +- 4 files changed, 149 insertions(+), 80 deletions(-) diff --git a/specifyweb/workbench/upload/tests/example_plan.py b/specifyweb/workbench/upload/tests/example_plan.py index 7aabb873bcb..fcc3277aa54 100644 --- a/specifyweb/workbench/upload/tests/example_plan.py +++ b/specifyweb/workbench/upload/tests/example_plan.py @@ -45,12 +45,14 @@ 'name': 'Species', 'author': 'Species Author', }, + treeId = 3 ), 'Subspecies': dict( treeNodeCols = { 'name': 'Subspecies', 'author': 'Subspecies Author', }, + treeId = 3 ), } )} diff --git a/specifyweb/workbench/upload/treerecord.py b/specifyweb/workbench/upload/treerecord.py index 55f77413e18..0aa77872cdf 100644 --- a/specifyweb/workbench/upload/treerecord.py +++ b/specifyweb/workbench/upload/treerecord.py @@ -43,7 +43,8 @@ def _get_treedef_id( base_treedef_id: Optional[int], ) -> int: filter_kwargs = self._build_filter_kwargs(rank_name, treedef_id) - tree_model = getattr(models, tree.lower().title() + 'treedefitem') + # tree_model = getattr(models, tree.lower().title() + 'treedefitem') + tree_model = get_tree_model(tree) treedefitems = tree_model.objects.filter(**filter_kwargs) if not treedefitems.exists(): @@ -75,7 +76,8 @@ def _filter_by_base_treedef_id(self, treedefitems, rank_name: str, base_treedef_ return treedefitems def check_rank(self) -> bool: - tree_model = getattr(models, self.tree.lower().title() + 'treedefitem') + # tree_model = getattr(models, self.tree.lower().title() + 'treedefitem') + tree_model = get_tree_model(self.tree) rank = tree_model.objects.filter(name=self.rank_name, treedef_id=self.treedef_id) return rank.exists() and rank.count() == 1 @@ -91,6 +93,14 @@ def tree_rank_record(self) -> 'TreeRankRecord': }, "Tree is required" return TreeRankRecord(self.rank_name, self.treedef_id) +def get_tree_model(tree: str): + return getattr(models, tree.lower().title() + 'treedefitem') + +# def is_rank_identifiable(rank: str, tree: str, treedef_id: int) -> bool: +# tree_model = get_tree_model(tree) +# results = tree_model.objects.filter(name=rank, treedef_id=treedef_id) +# return results.exists() and results.count() == 1 + class TreeRankRecord(NamedTuple): rank_name: str treedef_id: int @@ -112,8 +122,6 @@ def validate_rank(self, tree) -> None: class TreeRecord(NamedTuple): name: str - # ranks: Dict[str, Dict[str, ColumnOptions]] - # ranks: Dict[Union[str, Tuple[str, int]], Dict[str, ColumnOptions]] ranks: Dict[Union[str, TreeRankRecord], Dict[str, ColumnOptions]] base_treedef_id: Optional[int] = None @@ -125,21 +133,21 @@ def get_cols(self) -> Set[str]: return {col.column for r in self.ranks.values() for col in r.values() if hasattr(col, 'column')} def to_json(self) -> Dict: - result = { - "ranks": { - rank.rank_name if isinstance(rank, TreeRankRecord) else rank: ( - cols["name"].to_json() - if len(cols) == 1 - else dict( - treeNodeCols={ - k: v.to_json() if hasattr(v, "to_json") else v - for k, v in cols.items() - } - ) - ) - for rank, cols in self.ranks.items() - }, - } + # result: Dict[str, Union[str, int, Dict[str, Any]]] = {"ranks": {}} + result = {"ranks": {}} # type: ignore + + for rank, cols in self.ranks.items(): + rank_key = rank.rank_name if isinstance(rank, TreeRankRecord) else rank + treeNodeCols = {k: v.to_json() if hasattr(v, "to_json") else v for k, v in cols.items()} + + if len(cols) == 1: + result["ranks"][rank_key] = treeNodeCols["name"] + else: + rank_data = {"treeNodeCols": treeNodeCols} + if isinstance(rank, TreeRankRecord): + rank_data["treeId"] = rank.treedef_id # type: ignore + result["ranks"][rank_key] = rank_data + return {'treeRecord': result} def unparse(self) -> Dict: diff --git a/specifyweb/workbench/upload/upload_plan_schema.py b/specifyweb/workbench/upload/upload_plan_schema.py index 7c50ae8f0bb..14a246dd912 100644 --- a/specifyweb/workbench/upload/upload_plan_schema.py +++ b/specifyweb/workbench/upload/upload_plan_schema.py @@ -3,12 +3,10 @@ import logging from specifyweb.specify.datamodel import datamodel, Table, Relationship -from specifyweb.specify.load_datamodel import DoesNotExistError -from specifyweb.specify.models import Taxontreedefitem, Taxontreedef from .upload_table import DeferredScopeUploadTable, UploadTable, OneToOneTable, MustMatchTable from .tomany import ToManyRecord -from .treerecord import TreeRank, TreeRankRecord, TreeRecord, MustMatchTreeRecord +from .treerecord import TreeRank, TreeRankRecord, TreeRecord, MustMatchTreeRecord, get_tree_model from .uploadable import Uploadable from .column_options import ColumnOptions from .scoping import DEFERRED_SCOPING @@ -37,7 +35,7 @@ }, ] }, - "taxonTreeId": { + "treeId": { "oneOf": [ { "type": "integer" }, { "type": "null" } @@ -100,7 +98,7 @@ 'type': 'object', 'properties': { 'treeNodeCols': { '$ref': '#/definitions/treeNodeCols' }, - 'taxonTreeId': { '$ref': '#definitions/taxonTreeId'} + 'treeId': { '$ref': '#definitions/treeId'} }, 'required': [ 'treeNodeCols' ], 'additionalProperties': False @@ -150,7 +148,7 @@ ], }, - 'taxonTreeId': { + 'treeId': { "oneOf": [ { "type": "integer" }, { "type": "null" } @@ -235,8 +233,8 @@ def parse_plan_with_basetable(collection, to_parse: Dict) -> Tuple[Table, Uploadable]: base_table = datamodel.get_table_strict(to_parse['baseTableName']) extra = {} - if 'taxonTreeId' in to_parse.keys(): - extra['treedefid'] = to_parse['taxonTreeId'] + if 'treeId' in to_parse.keys(): + extra['treedefid'] = to_parse['treeId'] return base_table, parse_uploadable(collection, base_table, to_parse['uploadable'], extra) def parse_plan(collection, to_parse: Dict) -> Uploadable: @@ -244,7 +242,7 @@ def parse_plan(collection, to_parse: Dict) -> Uploadable: def parse_uploadable(collection, table: Table, to_parse: Dict, extra: Dict = {}) -> Uploadable: if 'uploadTable' in to_parse: - return parse_upload_table(collection, table, to_parse['uploadTable']) + return parse_upload_table(collection, table, to_parse['uploadTable'], extra) if 'oneToOneTable' in to_parse: return OneToOneTable(*parse_upload_table(collection, table, to_parse['oneToOneTable'])) @@ -263,7 +261,7 @@ def parse_uploadable(collection, table: Table, to_parse: Dict, extra: Dict = {}) raise ValueError('unknown uploadable type') -def parse_upload_table(collection, table: Table, to_parse: Dict) -> UploadTable: +def parse_upload_table(collection, table: Table, to_parse: Dict, extra: Dict = {}) -> UploadTable: def rel_table(key: str) -> Table: return datamodel.get_table_strict(table.get_relationship(key).relatedModelName) @@ -288,7 +286,7 @@ def defer_scope_upload_table(default_collection, table: Table, to_parse: Dict, d for key, to_one in to_parse['toOne'].items() }, toMany={ - key: [parse_to_many_record(default_collection, rel_table(key), record) for record in to_manys] + key: [parse_to_many_record(default_collection, rel_table(key), records) for record in to_manys] for key, to_manys in to_parse['toMany'].items() } ) @@ -305,7 +303,7 @@ def defer_scope_upload_table(default_collection, table: Table, to_parse: Dict, d for key, to_one in to_parse['toOne'].items() }, toMany={ - key: [parse_to_many_record(collection, rel_table(key), record) for record in to_manys] + key: [parse_to_many_record(collection, rel_table(key), record, extra) for record in to_manys] for key, to_manys in to_parse['toMany'].items() } ) @@ -313,6 +311,7 @@ def defer_scope_upload_table(default_collection, table: Table, to_parse: Dict, d def parse_tree_record(collection, table: Table, to_parse: Dict, base_treedefid: Optional[int] = None) -> TreeRecord: ranks: Dict[Union[str, TreeRankRecord], Dict[str, ColumnOptions]] = {} for rank, name_or_cols in to_parse['ranks'].items(): + treedefid = None if isinstance(name_or_cols, str): rank_name = name_or_cols parsed_cols = {'name': parse_column_options(name_or_cols)} @@ -320,20 +319,29 @@ def parse_tree_record(collection, table: Table, to_parse: Dict, base_treedefid: tree_node_cols = {k: parse_column_options(v) for k, v in name_or_cols['treeNodeCols'].items()} rank_name = name_or_cols['treeNodeCols']['name'] parsed_cols = tree_node_cols + if 'treeId' in name_or_cols: + treedefid = name_or_cols['treeId'] - treedefid = get_treedef_id(rank_name, False, base_treedefid) - tree_rank_record = TreeRank(rank, table.name, treedefid, base_treedefid).tree_rank_record() + # if treedefid is None: + # treedefid = get_treedef_id(rank_name, table.name, False, base_treedefid) + tree_rank_record = TreeRank(rank_name, table.name, treedefid, base_treedefid).tree_rank_record() ranks[tree_rank_record] = parsed_cols + if not isinstance(name_or_cols, str): # TODO: Make better + to_parse['ranks'][rank]['treeId'] = tree_rank_record.treedef_id + + if base_treedefid is None: + base_treedefid = tree_rank_record.treedef_id for rank, cols in ranks.items(): assert 'name' in cols, to_parse - if base_treedefid is None: - for tree_rank_record, cols in ranks.items(): # type: ignore - treedefid = get_treedef_id(cols['name'].column, False, base_treedefid) - if treedefid is not None: - base_treedefid = treedefid - break + # if base_treedefid is None: + # for tree_rank_record, cols in ranks.items(): # type: ignore + # # treedefid = get_treedef_id(cols['name'].column, table.name, False, base_treedefid) + # treedefid = + # if treedefid is not None: + # base_treedefid = treedefid + # break return TreeRecord( name=table.django_name, @@ -341,47 +349,100 @@ def parse_tree_record(collection, table: Table, to_parse: Dict, base_treedefid: base_treedef_id=base_treedefid ) -def find_treedef( - rank_name: str, treedef_id: Optional[int] = None, is_adjusting: bool = False -) -> Optional["Taxontreedef"]: - filter_kwargs = {'name': rank_name} - if treedef_id is not None: - filter_kwargs['treedef_id'] = treedef_id # type: ignore - ranks = Taxontreedefitem.objects.filter(**filter_kwargs) - if ranks.count() == 0: - return None - elif ranks.count() > 1 and is_adjusting and treedef_id is not None: - raise ValueError(f"Multiple treedefitems with name {rank_name} and treedef_id {treedef_id}") +# def find_treedef(rank_name: str, tree: str, treedef_id: Optional[int] = None, is_adjusting: bool = False): +# tree_model = get_tree_model(tree) +# filter_kwargs = {'name': rank_name} +# if treedef_id is not None: +# filter_kwargs['treedef_id'] = treedef_id # type: ignore +# ranks = tree_model.objects.filter(**filter_kwargs) +# if ranks.count() == 0: +# return None +# elif ranks.count() > 1 and is_adjusting and treedef_id is not None: +# raise ValueError(f"Multiple treedefitems with name {rank_name} and treedef_id {treedef_id}") + +# first_rank = ranks.first() +# if first_rank is None: +# return None + +# return first_rank.treedef + +# def get_treedef_id_old( +# rank_name: str, tree: str, is_adjusting: bool, base_treedef_id: Optional[int] = None +# ) -> Optional[int]: +# for treedef_id in [base_treedef_id, None]: +# treedef = find_treedef(rank_name, tree, treedef_id, is_adjusting) +# if treedef: +# return treedef.id + +# if is_adjusting: +# raise ValueError(f"Could not find treedefitem with name {rank_name}") +# return None + +# def get_treedef_id( +# rank_name: str, tree: str, is_adjusting: bool, base_treedef_id: Optional[int] = None +# ) -> Optional[int]: +# treedef = find_treedef(rank_name, tree, base_treedef_id, False) + +# if treedef is None: +# treedef = find_treedef(rank_name, tree, None, is_adjusting) + +# if is_adjusting: +# raise ValueError(f"Could not find treedefitem with name {rank_name}") +# return None + +# def adjust_upload_plan(plan: Dict, collection) -> Dict: +# if ( +# "uploadable" not in plan +# or "treeRecord" not in plan["uploadable"] +# or "ranks" not in plan["uploadable"]["treeRecord"] +# ): +# return plan + +# base_table = datamodel.get_table_strict(plan['baseTableName']) +# uploadable = parse_uploadable(collection, base_table, plan['uploadable']) +# tree_table_name = plan['uploadable']['uploadTable']['toMany'] + +# # determinations = plan['uploadable']['uploadTable']['toMany']['determinations'] +# # for determination in determinations: +# tree = plan['uploadable']['uploadTable']['toMany']['determinations'][0]['toOne'].keys()[0] - first_rank = ranks.first() - if first_rank is None: - return None +# base_treedef_id = plan.get('treeId', None) +# tree = 'Taxon' # tree = plan['uploadable'] +# for rank, name_or_cols in plan['uploadable']['treeRecord']['ranks'].items(): +# if isinstance(name_or_cols, dict): +# rank_name = name_or_cols['treeNodeCols']['name'] +# if 'treedefid' not in name_or_cols: +# # name_or_cols['treeId'] = get_treedef_id(rank_name, tree, True, base_treedef_id) +# name_or_cols['treeId'] = TreeRank(rank_name, tree, None, base_treedef_id).treedef_id + +# return plan + +def adjust_upload_plan(plan: Dict, collection): + base_table = datamodel.get_table_strict(plan['baseTableName']) + extra = {} + if 'treeId' in plan.keys(): + extra['treedefid'] = plan['treeId'] - return first_rank.treedef - -def get_treedef_id(rank_name: str, is_adjusting: bool, base_treedef_id: Optional[int] = None) -> Optional[int]: - for treedef_id in [base_treedef_id, None]: - treedef = find_treedef(rank_name, treedef_id, is_adjusting) - if treedef: - return treedef.id - - if is_adjusting: - raise ValueError(f"Could not find treedefitem with name {rank_name}") - return None - -def adjust_upload_plan(plan: Dict) -> Dict: - if plan['baseTableName'] == 'taxon': - base_treedef_id = plan.get('taxonTreeId', None) - for rank, name_or_cols in plan['uploadable']['treeRecord']['ranks'].items(): - if isinstance(name_or_cols, dict): - rank_name = name_or_cols['treeNodeCols']['name'] - if 'treedefid' not in name_or_cols: - # name_or_cols['treedefid'] = str(get_treedef_id(rank_name, True, base_treedef_id)) - name_or_cols['taxonTreeId'] = get_treedef_id(rank_name, True, base_treedef_id) - + if 'treeRecord' in plan: + treedefid = None + if 'treedefid' in extra.keys(): + treedefid = int(extra['treedefid']) + + for rank, name_or_cols in plan['ranks'].items(): + treedefid = None + if not isinstance(name_or_cols, str): + tree_node_cols = {k: parse_column_options(v) for k, v in name_or_cols['treeNodeCols'].items()} + rank_name = name_or_cols['treeNodeCols']['name'] + parsed_cols = tree_node_cols + if 'treeId' in name_or_cols: + treedefid = name_or_cols['treeId'] + # tree_rank_record = TreeRank(rank, table.name, treedefid, base_treedefid).tree_rank_record() + # plan['ranks'][rank]['treeId'] = tree_rank_record.treedef_id + return plan + -def parse_to_many_record(collection, table: Table, to_parse: Dict) -> ToManyRecord: +def parse_to_many_record(collection, table: Table, to_parse: Dict, extra: Dict = {}) -> ToManyRecord: def rel_table(key: str) -> Table: return datamodel.get_table_strict(table.get_relationship(key).relatedModelName) @@ -391,7 +452,7 @@ def rel_table(key: str) -> Table: wbcols={k: parse_column_options(v) for k,v in to_parse['wbcols'].items()}, static=to_parse['static'], toOne={ - key: parse_uploadable(collection, rel_table(key), to_one) + key: parse_uploadable(collection, rel_table(key), to_one, extra) for key, to_one in to_parse['toOne'].items() }, ) diff --git a/specifyweb/workbench/views.py b/specifyweb/workbench/views.py index 381df068b5e..cd852c850e1 100644 --- a/specifyweb/workbench/views.py +++ b/specifyweb/workbench/views.py @@ -1,6 +1,5 @@ import json import logging -import re from typing import List, Optional from uuid import uuid4 @@ -484,8 +483,7 @@ def dataset(request, ds: models.Spdataset) -> http.HttpResponse: parsed_plan = upload_plan_schema.parse_plan(request.specify_collection, plan) new_cols = parsed_plan.get_cols() - set(ds.columns) - if plan['baseTableName'] == 'taxon': - plan = upload_plan_schema.adjust_upload_plan(plan) + # plan = upload_plan_schema.adjust_upload_plan(plan, request.specify_collection) if new_cols: ncols = len(ds.columns) ds.columns += list(new_cols) From cc62e3fae0fa20bf06cec0743801b8fbee1d335e Mon Sep 17 00:00:00 2001 From: alec_dev Date: Fri, 23 Aug 2024 15:18:10 -0500 Subject: [PATCH 26/78] remove adjust_upload_plan --- specifyweb/workbench/upload/treerecord.py | 7 - .../workbench/upload/upload_plan_schema.py | 130 ++---------------- 2 files changed, 13 insertions(+), 124 deletions(-) diff --git a/specifyweb/workbench/upload/treerecord.py b/specifyweb/workbench/upload/treerecord.py index 0aa77872cdf..b4e8e4c6d9b 100644 --- a/specifyweb/workbench/upload/treerecord.py +++ b/specifyweb/workbench/upload/treerecord.py @@ -43,7 +43,6 @@ def _get_treedef_id( base_treedef_id: Optional[int], ) -> int: filter_kwargs = self._build_filter_kwargs(rank_name, treedef_id) - # tree_model = getattr(models, tree.lower().title() + 'treedefitem') tree_model = get_tree_model(tree) treedefitems = tree_model.objects.filter(**filter_kwargs) @@ -76,7 +75,6 @@ def _filter_by_base_treedef_id(self, treedefitems, rank_name: str, base_treedef_ return treedefitems def check_rank(self) -> bool: - # tree_model = getattr(models, self.tree.lower().title() + 'treedefitem') tree_model = get_tree_model(self.tree) rank = tree_model.objects.filter(name=self.rank_name, treedef_id=self.treedef_id) return rank.exists() and rank.count() == 1 @@ -96,11 +94,6 @@ def tree_rank_record(self) -> 'TreeRankRecord': def get_tree_model(tree: str): return getattr(models, tree.lower().title() + 'treedefitem') -# def is_rank_identifiable(rank: str, tree: str, treedef_id: int) -> bool: -# tree_model = get_tree_model(tree) -# results = tree_model.objects.filter(name=rank, treedef_id=treedef_id) -# return results.exists() and results.count() == 1 - class TreeRankRecord(NamedTuple): rank_name: str treedef_id: int diff --git a/specifyweb/workbench/upload/upload_plan_schema.py b/specifyweb/workbench/upload/upload_plan_schema.py index 14a246dd912..070fcfa66be 100644 --- a/specifyweb/workbench/upload/upload_plan_schema.py +++ b/specifyweb/workbench/upload/upload_plan_schema.py @@ -286,7 +286,7 @@ def defer_scope_upload_table(default_collection, table: Table, to_parse: Dict, d for key, to_one in to_parse['toOne'].items() }, toMany={ - key: [parse_to_many_record(default_collection, rel_table(key), records) for record in to_manys] + key: [parse_to_many_record(default_collection, rel_table(key), record) for record in to_manys] for key, to_manys in to_parse['toMany'].items() } ) @@ -309,24 +309,21 @@ def defer_scope_upload_table(default_collection, table: Table, to_parse: Dict, d ) def parse_tree_record(collection, table: Table, to_parse: Dict, base_treedefid: Optional[int] = None) -> TreeRecord: + def parse_rank(name_or_cols): + if isinstance(name_or_cols, str): + return name_or_cols, {'name': parse_column_options(name_or_cols)}, None + tree_node_cols = {k: parse_column_options(v) for k, v in name_or_cols['treeNodeCols'].items()} + rank_name = name_or_cols['treeNodeCols']['name'] + treedefid = name_or_cols.get('treeId') + return rank_name, tree_node_cols, treedefid + ranks: Dict[Union[str, TreeRankRecord], Dict[str, ColumnOptions]] = {} for rank, name_or_cols in to_parse['ranks'].items(): - treedefid = None - if isinstance(name_or_cols, str): - rank_name = name_or_cols - parsed_cols = {'name': parse_column_options(name_or_cols)} - else: - tree_node_cols = {k: parse_column_options(v) for k, v in name_or_cols['treeNodeCols'].items()} - rank_name = name_or_cols['treeNodeCols']['name'] - parsed_cols = tree_node_cols - if 'treeId' in name_or_cols: - treedefid = name_or_cols['treeId'] - - # if treedefid is None: - # treedefid = get_treedef_id(rank_name, table.name, False, base_treedefid) - tree_rank_record = TreeRank(rank_name, table.name, treedefid, base_treedefid).tree_rank_record() + rank_name, parsed_cols, treedefid = parse_rank(name_or_cols) + tree_rank_record = TreeRank(rank, table.name, treedefid, base_treedefid).tree_rank_record() ranks[tree_rank_record] = parsed_cols - if not isinstance(name_or_cols, str): # TODO: Make better + + if not isinstance(name_or_cols, str): to_parse['ranks'][rank]['treeId'] = tree_rank_record.treedef_id if base_treedefid is None: @@ -335,113 +332,12 @@ def parse_tree_record(collection, table: Table, to_parse: Dict, base_treedefid: for rank, cols in ranks.items(): assert 'name' in cols, to_parse - # if base_treedefid is None: - # for tree_rank_record, cols in ranks.items(): # type: ignore - # # treedefid = get_treedef_id(cols['name'].column, table.name, False, base_treedefid) - # treedefid = - # if treedefid is not None: - # base_treedefid = treedefid - # break - return TreeRecord( name=table.django_name, ranks=ranks, base_treedef_id=base_treedefid ) -# def find_treedef(rank_name: str, tree: str, treedef_id: Optional[int] = None, is_adjusting: bool = False): -# tree_model = get_tree_model(tree) -# filter_kwargs = {'name': rank_name} -# if treedef_id is not None: -# filter_kwargs['treedef_id'] = treedef_id # type: ignore -# ranks = tree_model.objects.filter(**filter_kwargs) -# if ranks.count() == 0: -# return None -# elif ranks.count() > 1 and is_adjusting and treedef_id is not None: -# raise ValueError(f"Multiple treedefitems with name {rank_name} and treedef_id {treedef_id}") - -# first_rank = ranks.first() -# if first_rank is None: -# return None - -# return first_rank.treedef - -# def get_treedef_id_old( -# rank_name: str, tree: str, is_adjusting: bool, base_treedef_id: Optional[int] = None -# ) -> Optional[int]: -# for treedef_id in [base_treedef_id, None]: -# treedef = find_treedef(rank_name, tree, treedef_id, is_adjusting) -# if treedef: -# return treedef.id - -# if is_adjusting: -# raise ValueError(f"Could not find treedefitem with name {rank_name}") -# return None - -# def get_treedef_id( -# rank_name: str, tree: str, is_adjusting: bool, base_treedef_id: Optional[int] = None -# ) -> Optional[int]: -# treedef = find_treedef(rank_name, tree, base_treedef_id, False) - -# if treedef is None: -# treedef = find_treedef(rank_name, tree, None, is_adjusting) - -# if is_adjusting: -# raise ValueError(f"Could not find treedefitem with name {rank_name}") -# return None - -# def adjust_upload_plan(plan: Dict, collection) -> Dict: -# if ( -# "uploadable" not in plan -# or "treeRecord" not in plan["uploadable"] -# or "ranks" not in plan["uploadable"]["treeRecord"] -# ): -# return plan - -# base_table = datamodel.get_table_strict(plan['baseTableName']) -# uploadable = parse_uploadable(collection, base_table, plan['uploadable']) -# tree_table_name = plan['uploadable']['uploadTable']['toMany'] - -# # determinations = plan['uploadable']['uploadTable']['toMany']['determinations'] -# # for determination in determinations: -# tree = plan['uploadable']['uploadTable']['toMany']['determinations'][0]['toOne'].keys()[0] - -# base_treedef_id = plan.get('treeId', None) -# tree = 'Taxon' # tree = plan['uploadable'] -# for rank, name_or_cols in plan['uploadable']['treeRecord']['ranks'].items(): -# if isinstance(name_or_cols, dict): -# rank_name = name_or_cols['treeNodeCols']['name'] -# if 'treedefid' not in name_or_cols: -# # name_or_cols['treeId'] = get_treedef_id(rank_name, tree, True, base_treedef_id) -# name_or_cols['treeId'] = TreeRank(rank_name, tree, None, base_treedef_id).treedef_id - -# return plan - -def adjust_upload_plan(plan: Dict, collection): - base_table = datamodel.get_table_strict(plan['baseTableName']) - extra = {} - if 'treeId' in plan.keys(): - extra['treedefid'] = plan['treeId'] - - if 'treeRecord' in plan: - treedefid = None - if 'treedefid' in extra.keys(): - treedefid = int(extra['treedefid']) - - for rank, name_or_cols in plan['ranks'].items(): - treedefid = None - if not isinstance(name_or_cols, str): - tree_node_cols = {k: parse_column_options(v) for k, v in name_or_cols['treeNodeCols'].items()} - rank_name = name_or_cols['treeNodeCols']['name'] - parsed_cols = tree_node_cols - if 'treeId' in name_or_cols: - treedefid = name_or_cols['treeId'] - # tree_rank_record = TreeRank(rank, table.name, treedefid, base_treedefid).tree_rank_record() - # plan['ranks'][rank]['treeId'] = tree_rank_record.treedef_id - - return plan - - def parse_to_many_record(collection, table: Table, to_parse: Dict, extra: Dict = {}) -> ToManyRecord: def rel_table(key: str) -> Table: From e78e950dcd8e90c3e2fdd23e4940c34ffa6c5e32 Mon Sep 17 00:00:00 2001 From: alec_dev Date: Fri, 23 Aug 2024 17:35:40 -0500 Subject: [PATCH 27/78] avoid schema adjustment for now --- specifyweb/workbench/upload/tests/example_plan.py | 1 + specifyweb/workbench/upload/upload_plan_schema.py | 5 +++-- specifyweb/workbench/views.py | 1 - 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/specifyweb/workbench/upload/tests/example_plan.py b/specifyweb/workbench/upload/tests/example_plan.py index fcc3277aa54..d3b21a7bae8 100644 --- a/specifyweb/workbench/upload/tests/example_plan.py +++ b/specifyweb/workbench/upload/tests/example_plan.py @@ -2,6 +2,7 @@ from ..tomany import ToManyRecord from ..treerecord import TreeRecord from ..upload_plan_schema import parse_column_options +# from specifyweb.specify import models as spmodels json = dict( baseTableName = 'Collectionobject', diff --git a/specifyweb/workbench/upload/upload_plan_schema.py b/specifyweb/workbench/upload/upload_plan_schema.py index 070fcfa66be..3c2f4171038 100644 --- a/specifyweb/workbench/upload/upload_plan_schema.py +++ b/specifyweb/workbench/upload/upload_plan_schema.py @@ -323,8 +323,9 @@ def parse_rank(name_or_cols): tree_rank_record = TreeRank(rank, table.name, treedefid, base_treedefid).tree_rank_record() ranks[tree_rank_record] = parsed_cols - if not isinstance(name_or_cols, str): - to_parse['ranks'][rank]['treeId'] = tree_rank_record.treedef_id + # Update the schema with the treeId + # if not isinstance(name_or_cols, str): + # to_parse['ranks'][rank]['treeId'] = tree_rank_record.treedef_id if base_treedefid is None: base_treedefid = tree_rank_record.treedef_id diff --git a/specifyweb/workbench/views.py b/specifyweb/workbench/views.py index cd852c850e1..a1f94d3e2d0 100644 --- a/specifyweb/workbench/views.py +++ b/specifyweb/workbench/views.py @@ -483,7 +483,6 @@ def dataset(request, ds: models.Spdataset) -> http.HttpResponse: parsed_plan = upload_plan_schema.parse_plan(request.specify_collection, plan) new_cols = parsed_plan.get_cols() - set(ds.columns) - # plan = upload_plan_schema.adjust_upload_plan(plan, request.specify_collection) if new_cols: ncols = len(ds.columns) ds.columns += list(new_cols) From 1ab212c90f47320539880df2bf016c383bbfa586 Mon Sep 17 00:00:00 2001 From: alec_dev Date: Mon, 26 Aug 2024 10:48:02 -0500 Subject: [PATCH 28/78] fix _get_treedef_id --- .../workbench/upload/tests/example_plan.py | 4 ++-- specifyweb/workbench/upload/treerecord.py | 16 +++++++++------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/specifyweb/workbench/upload/tests/example_plan.py b/specifyweb/workbench/upload/tests/example_plan.py index d3b21a7bae8..0dd9f50fbe2 100644 --- a/specifyweb/workbench/upload/tests/example_plan.py +++ b/specifyweb/workbench/upload/tests/example_plan.py @@ -46,14 +46,14 @@ 'name': 'Species', 'author': 'Species Author', }, - treeId = 3 + treeId = 142 # TODO: Set this dynamically ), 'Subspecies': dict( treeNodeCols = { 'name': 'Subspecies', 'author': 'Subspecies Author', }, - treeId = 3 + treeId = 142 # TODO: Set this dynamically ), } )} diff --git a/specifyweb/workbench/upload/treerecord.py b/specifyweb/workbench/upload/treerecord.py index b4e8e4c6d9b..bd257e3e84e 100644 --- a/specifyweb/workbench/upload/treerecord.py +++ b/specifyweb/workbench/upload/treerecord.py @@ -42,12 +42,14 @@ def _get_treedef_id( treedef_id: Optional[int], base_treedef_id: Optional[int], ) -> int: - filter_kwargs = self._build_filter_kwargs(rank_name, treedef_id) tree_model = get_tree_model(tree) - treedefitems = tree_model.objects.filter(**filter_kwargs) - if not treedefitems.exists(): - raise ValueError(f"No treedefitems found for rank {rank_name}") + filter_kwargs = self._build_filter_kwargs(rank_name, treedef_id) + treedefitems = tree_model.objects.filter(**filter_kwargs) + + if not treedefitems.exists() and treedef_id is not None: + filter_kwargs = self._build_filter_kwargs(rank_name) + treedefitems = tree_model.objects.filter(**filter_kwargs) if treedefitems.count() > 1: treedefitems = self._filter_by_base_treedef_id(treedefitems, rank_name, base_treedef_id) @@ -58,7 +60,7 @@ def _get_treedef_id( return first_item.treedef_id - def _build_filter_kwargs(self, rank_name: str, treedef_id: Optional[int]) -> Dict[str, Any]: + def _build_filter_kwargs(self, rank_name: str, treedef_id: Optional[int] = None) -> Dict[str, Any]: filter_kwargs = {'name': rank_name} if treedef_id is not None: filter_kwargs['treedef_id'] = treedef_id # type: ignore @@ -105,7 +107,7 @@ def to_json(self) -> Dict: } def to_key(self) -> Tuple[str, int]: - return (self.rank_name, self.treedef_id) + return (self.rank_name, self.treedef_id) # Check this line in unit test def check_rank(self, tree: str) -> bool: return TreeRank(self.rank_name, tree, self.treedef_id).check_rank() @@ -130,7 +132,7 @@ def to_json(self) -> Dict: result = {"ranks": {}} # type: ignore for rank, cols in self.ranks.items(): - rank_key = rank.rank_name if isinstance(rank, TreeRankRecord) else rank + rank_key = rank.rank_name if isinstance(rank, TreeRankRecord) else rank # Check this line in unit test treeNodeCols = {k: v.to_json() if hasattr(v, "to_json") else v for k, v in cols.items()} if len(cols) == 1: From 03b678413a0ac332375194592f9f60094907ab42 Mon Sep 17 00:00:00 2001 From: alec_dev Date: Mon, 26 Aug 2024 11:22:40 -0500 Subject: [PATCH 29/78] set treeId dynamically in unit tests --- specifyweb/workbench/upload/tests/example_plan.py | 2 -- specifyweb/workbench/upload/tests/testschema.py | 10 ++++++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/specifyweb/workbench/upload/tests/example_plan.py b/specifyweb/workbench/upload/tests/example_plan.py index 0dd9f50fbe2..9d2d99433af 100644 --- a/specifyweb/workbench/upload/tests/example_plan.py +++ b/specifyweb/workbench/upload/tests/example_plan.py @@ -46,14 +46,12 @@ 'name': 'Species', 'author': 'Species Author', }, - treeId = 142 # TODO: Set this dynamically ), 'Subspecies': dict( treeNodeCols = { 'name': 'Subspecies', 'author': 'Subspecies Author', }, - treeId = 142 # TODO: Set this dynamically ), } )} diff --git a/specifyweb/workbench/upload/tests/testschema.py b/specifyweb/workbench/upload/tests/testschema.py index 3b9d1117e68..111776a8e93 100644 --- a/specifyweb/workbench/upload/tests/testschema.py +++ b/specifyweb/workbench/upload/tests/testschema.py @@ -14,10 +14,19 @@ from .base import UploadTestsBase from . import example_plan +def set_plan_treeId(): + # Set the treeId in example_plan.json dynamically, so that the unit test doesn't depend on the static treeId to always be the same. + from specifyweb.specify.models import Taxontreedefitem + tree_id = Taxontreedefitem.objects.filter(name='Species').first().treedef_id + example_plan_ranks = example_plan.json['uploadable']['uploadTable']['toMany']['determinations'][0]['toOne']['taxon']['treeRecord']['ranks'] + for rank in ['Species', 'Subspecies']: + example_plan_ranks[rank]['treeId'] = tree_id + class SchemaTests(UploadTestsBase): maxDiff = None def test_schema_parsing(self) -> None: + set_plan_treeId() Draft7Validator.check_schema(schema) validate(example_plan.json, schema) plan = parse_plan(self.collection, example_plan.json).apply_scoping(self.collection) @@ -26,6 +35,7 @@ def test_schema_parsing(self) -> None: self.assertEqual(repr(plan), repr(self.example_plan)) def test_unparsing(self) -> None: + set_plan_treeId() self.assertEqual(example_plan.json, parse_plan(self.collection, example_plan.json).unparse()) def test_reject_internal_tree_columns(self) -> None: From 94154e32299e613a672acdb95433ac0d6327e3df Mon Sep 17 00:00:00 2001 From: melton-jason Date: Wed, 28 Aug 2024 09:46:19 -0500 Subject: [PATCH 30/78] Implement choosing a specific treeDef in mappers --- .../js_src/lib/components/DataModel/schema.ts | 7 +- .../components/InitialContext/treeRanks.ts | 13 +- .../lib/components/WbPlanView/Mapper.tsx | 15 -- .../lib/components/WbPlanView/Wrapped.tsx | 107 ++-------- .../__tests__/mappingHelpers.test.ts | 10 +- .../lib/components/WbPlanView/helpers.ts | 5 +- .../components/WbPlanView/mappingHelpers.ts | 37 +++- .../lib/components/WbPlanView/navigator.ts | 192 +++++++++++------- .../components/WbPlanView/navigatorSpecs.ts | 5 + .../WbPlanView/uploadPlanBuilder.ts | 5 +- .../components/WbPlanView/uploadPlanParser.ts | 1 - .../frontend/js_src/lib/localization/query.ts | 3 + .../js_src/lib/localization/workbench.ts | 3 - 13 files changed, 188 insertions(+), 215 deletions(-) diff --git a/specifyweb/frontend/js_src/lib/components/DataModel/schema.ts b/specifyweb/frontend/js_src/lib/components/DataModel/schema.ts index f1ed868f49c..a121851d82c 100644 --- a/specifyweb/frontend/js_src/lib/components/DataModel/schema.ts +++ b/specifyweb/frontend/js_src/lib/components/DataModel/schema.ts @@ -29,7 +29,8 @@ type Schema = { 'Institution' ]; readonly referenceSymbol: string; - readonly treeSymbol: string; + readonly treeDefinitionSymbol: string; + readonly treeRankSymbol: string; readonly fieldPartSeparator: string; }; @@ -60,8 +61,10 @@ const schemaBase: Writable = { // Prefix for -to-many indexes referenceSymbol: '#', + // Prefix for Tree Definitions + treeDefinitionSymbol: '%', // Prefix for tree ranks - treeSymbol: '$', + treeRankSymbol: '$', // Separator for partial fields (date parts in Query Builder) fieldPartSeparator: '-', }; diff --git a/specifyweb/frontend/js_src/lib/components/InitialContext/treeRanks.ts b/specifyweb/frontend/js_src/lib/components/InitialContext/treeRanks.ts index 44c6deeba2b..29c84c9da1c 100644 --- a/specifyweb/frontend/js_src/lib/components/InitialContext/treeRanks.ts +++ b/specifyweb/frontend/js_src/lib/components/InitialContext/treeRanks.ts @@ -118,11 +118,14 @@ export function getTreeDefinitions( tableName ); - return typeof treeDefinitionId === 'number' - ? specificTreeDefinitions.filter( - ({ definition }) => definition.id === treeDefinitionId - ) - : specificTreeDefinitions; + if (typeof treeDefinitionId === 'number') { + const resolvedDefinition = specificTreeDefinitions.find( + ({ definition }) => definition.id === treeDefinitionId + ); + return resolvedDefinition === undefined + ? specificTreeDefinitions + : [resolvedDefinition]; + } else return specificTreeDefinitions; } export function getTreeDefinitionItems( diff --git a/specifyweb/frontend/js_src/lib/components/WbPlanView/Mapper.tsx b/specifyweb/frontend/js_src/lib/components/WbPlanView/Mapper.tsx index 3125f322e3b..044dbca8ab3 100644 --- a/specifyweb/frontend/js_src/lib/components/WbPlanView/Mapper.tsx +++ b/specifyweb/frontend/js_src/lib/components/WbPlanView/Mapper.tsx @@ -24,15 +24,10 @@ import { className } from '../Atoms/className'; import { icons } from '../Atoms/Icons'; import { Link } from '../Atoms/Link'; import { LoadingContext, ReadOnlyContext } from '../Core/Contexts'; -import type { - FilterTablesByEndsWith, - SerializedResource, -} from '../DataModel/helperTypes'; import { strictGetTable } from '../DataModel/tables'; import type { Tables } from '../DataModel/types'; import { softFail } from '../Errors/Crash'; import { ErrorBoundary } from '../Errors/ErrorBoundary'; -import type { TreeInformation } from '../InitialContext/treeRanks'; import { TableIcon } from '../Molecules/TableIcon'; import { Layout } from './Header'; import { @@ -144,11 +139,6 @@ export function Mapper(props: { readonly changesMade: boolean; readonly lines: RA; readonly mustMatchPreferences: IR; - readonly taxonType?: string; - readonly treeDefinitions: - | readonly SerializedResource>[] - | undefined; - readonly handleTreeSelection: (treeName: string) => void; }): JSX.Element { const [state, dispatch] = React.useReducer( reducer, @@ -457,13 +447,9 @@ export function Mapper(props: { mustMatchPreferences: state.mustMatchPreferences, generateFieldData: 'all', spec: navigatorSpecs.wbPlanView, - taxonType: props.taxonType, - treeDefinitions: props.treeDefinitions, }), customSelectType: 'OPENED_LIST', onChange({ isDoubleClick, ...rest }) { - if (rest.index === 0 && rest.newTableName === 'Taxon') - props.handleTreeSelection(rest.newValue); if (isDoubleClick && mapButtonEnabled) dispatch({ type: 'MappingViewMapAction' }); else if (!isReadOnly) @@ -549,7 +535,6 @@ export function Mapper(props: { mustMatchPreferences: state.mustMatchPreferences, generateFieldData: 'all', spec: navigatorSpecs.wbPlanView, - taxonType: props.taxonType, }), }); diff --git a/specifyweb/frontend/js_src/lib/components/WbPlanView/Wrapped.tsx b/specifyweb/frontend/js_src/lib/components/WbPlanView/Wrapped.tsx index 5ffccd52b74..4f6e50edefa 100644 --- a/specifyweb/frontend/js_src/lib/components/WbPlanView/Wrapped.tsx +++ b/specifyweb/frontend/js_src/lib/components/WbPlanView/Wrapped.tsx @@ -9,19 +9,11 @@ import { useNavigate } from 'react-router-dom'; import type { LocalizedString } from 'typesafe-i18n'; import type { State } from 'typesafe-reducer'; -import { usePromise } from '../../hooks/useAsyncState'; import { useErrorContext } from '../../hooks/useErrorContext'; import { useLiveState } from '../../hooks/useLiveState'; -import { commonText } from '../../localization/common'; -import { wbText } from '../../localization/workbench'; -import { type IR, type RA, localized } from '../../utils/types'; -import { caseInsensitiveHash } from '../../utils/utils'; -import { Ul } from '../Atoms'; -import { Button } from '../Atoms/Button'; +import type { IR, RA } from '../../utils/types'; import type { Tables } from '../DataModel/types'; -import { treeRanksPromise } from '../InitialContext/treeRanks'; import { useTitle } from '../Molecules/AppTitle'; -import { Dialog } from '../Molecules/Dialog'; import { ProtectedAction } from '../Permissions/PermissionDenied'; import type { UploadResult } from '../WorkBench/resultsParser'; import { savePlan } from './helpers'; @@ -129,76 +121,24 @@ export function WbPlanView({ ); useErrorContext('state', state); - const [isTaxonTable, setIsTaxonTable] = React.useState(false); - - const [treeDefinitions] = usePromise(treeRanksPromise, true); - const definitionsForTreeTaxon = treeDefinitions - ? caseInsensitiveHash(treeDefinitions, 'Taxon') - : undefined; - const definitions = definitionsForTreeTaxon?.map( - ({ definition }) => definition - ); - - const [taxonType, setTaxonType] = React.useState(); - const handleTreeType = (taxonTreeDefinitionName: string): void => { - setTaxonType(taxonTreeDefinitionName); - setState({ - type: 'MappingState', - changesMade: true, - baseTableName: 'Taxon', - lines: getLinesFromHeaders({ - headers, - runAutoMapper: true, - baseTableName: 'Taxon', - }), - mustMatchPreferences: {}, - }); - }; - const taxonTree = definitionsForTreeTaxon?.find( - (tree) => tree.definition.name === taxonType - ); - const taxonTreeId = taxonTree?.definition.id; - const taxonCorrespondingId = definitionsForTreeTaxon?.find( - (tree) => tree.definition.id === uploadPlan?.taxonTreeId - ); - const taxonIdName = taxonCorrespondingId?.definition.name; - - const [treeSelected, setTreeSelected] = React.useState(); - - const handleTreeSelection = (treeName: string): void => { - const taxonCorresponding = definitionsForTreeTaxon?.find( - (tree) => tree.definition.name === treeName - ); - const taxonIdName = taxonCorresponding?.definition.name; - setTreeSelected(taxonIdName); - }; - const navigate = useNavigate(); return state.type === 'SelectBaseTable' ? ( navigate(`/specify/workbench/${dataset.id}/`)} - onSelected={ - (baseTableName): void => { - /* - * If (baseTableName === 'Taxon') { - * setIsTaxonTable(true); - * } else { - */ - setState({ - type: 'MappingState', - changesMade: true, + onSelected={(baseTableName): void => + setState({ + type: 'MappingState', + changesMade: true, + baseTableName, + lines: getLinesFromHeaders({ + headers, + runAutoMapper: true, baseTableName, - lines: getLinesFromHeaders({ - headers, - runAutoMapper: true, - baseTableName, - }), - mustMatchPreferences: {}, - }); - } - // } + }), + mustMatchPreferences: {}, + }) } onSelectTemplate={(uploadPlan, headers): void => setState({ @@ -208,36 +148,14 @@ export function WbPlanView({ }) } /> - {isTaxonTable && ( - {commonText.close()} - } - header={wbText.selectTree()} - onClose={(): void => setIsTaxonTable(false)} - > -
    - {definitions?.map(({ name }) => ( -
  • - handleTreeType(name)}> - {localized(name)} - -
    • - ))} -
- - )} ) : ( setState({ type: 'SelectBaseTable', @@ -249,7 +167,6 @@ export function WbPlanView({ baseTableName: state.baseTableName, lines, mustMatchPreferences, - taxonTreeId, }).then(() => navigate(`/specify/workbench/${dataset.id}/`)) } /> diff --git a/specifyweb/frontend/js_src/lib/components/WbPlanView/__tests__/mappingHelpers.test.ts b/specifyweb/frontend/js_src/lib/components/WbPlanView/__tests__/mappingHelpers.test.ts index d6df35e6dfb..866f9ec1520 100644 --- a/specifyweb/frontend/js_src/lib/components/WbPlanView/__tests__/mappingHelpers.test.ts +++ b/specifyweb/frontend/js_src/lib/components/WbPlanView/__tests__/mappingHelpers.test.ts @@ -16,7 +16,7 @@ theories(valueIsToManyIndex, [ [[`${schema.referenceSymbol}2`], true], [[`${schema.referenceSymbol}999`], true], [['collectionobject'], false], - [[`${schema.treeSymbol}Kingdom`], false], + [[`${schema.treeRankSymbol}Kingdom`], false], ]); theories(valueIsTreeRank, [ @@ -24,8 +24,8 @@ theories(valueIsTreeRank, [ [[`${schema.referenceSymbol}2`], false], [[`${schema.referenceSymbol}999`], false], [['collectionobject'], false], - [[`${schema.treeSymbol}Kingdom`], true], - [[`${schema.treeSymbol}County`], true], + [[`${schema.treeRankSymbol}Kingdom`], true], + [[`${schema.treeRankSymbol}County`], true], ]); theories(getNumberFromToManyIndex, [ @@ -36,8 +36,8 @@ theories(getNumberFromToManyIndex, [ ]); theories(getNameFromTreeRankName, [ - [[`${schema.treeSymbol}Kingdom`], 'Kingdom'], - [[`${schema.treeSymbol}County`], 'County'], + [[`${schema.treeRankSymbol}Kingdom`], 'Kingdom'], + [[`${schema.treeRankSymbol}County`], 'County'], ]); theories(findDuplicateMappings, [ diff --git a/specifyweb/frontend/js_src/lib/components/WbPlanView/helpers.ts b/specifyweb/frontend/js_src/lib/components/WbPlanView/helpers.ts index af69d9667e7..7b7dd213d92 100644 --- a/specifyweb/frontend/js_src/lib/components/WbPlanView/helpers.ts +++ b/specifyweb/frontend/js_src/lib/components/WbPlanView/helpers.ts @@ -42,13 +42,11 @@ export async function savePlan({ baseTableName, lines, mustMatchPreferences, - taxonTreeId, }: { readonly dataset: Dataset; readonly baseTableName: keyof Tables; readonly lines: RA; readonly mustMatchPreferences: IR; - readonly taxonTreeId?: number | undefined; }): Promise { const renamedLines = renameNewlyCreatedHeaders( baseTableName, @@ -68,8 +66,7 @@ export async function savePlan({ const uploadPlan = uploadPlanBuilder( baseTableName, renamedLines, - getMustMatchTables({ baseTableName, lines, mustMatchPreferences }), - taxonTreeId + getMustMatchTables({ baseTableName, lines, mustMatchPreferences }) ); const dataSetRequestUrl = `/api/workbench/dataset/${dataset.id}/`; diff --git a/specifyweb/frontend/js_src/lib/components/WbPlanView/mappingHelpers.ts b/specifyweb/frontend/js_src/lib/components/WbPlanView/mappingHelpers.ts index a5af31c19c9..dc65961182d 100644 --- a/specifyweb/frontend/js_src/lib/components/WbPlanView/mappingHelpers.ts +++ b/specifyweb/frontend/js_src/lib/components/WbPlanView/mappingHelpers.ts @@ -29,9 +29,20 @@ export const valueIsToManyIndex = (value: string | undefined): boolean => value?.slice(0, schema.referenceSymbol.length) === schema.referenceSymbol || false; +/** + * Returns whether a value is any special tree-related meta information + * (e.g. tree defintiion, tree rank) + */ +export const valueIsTreeMeta = (value: string | undefined): boolean => + valueIsTreeDefinition(value) || valueIsTreeRank(value); + +/** Returns whether a value is a tree definition name (e.x %Taxonomy) */ +export const valueIsTreeDefinition = (value: string | undefined): boolean => + value?.startsWith(schema.treeDefinitionSymbol) ?? false; + /** Returns whether a value is a tree rank name (e.x $Kingdom, $Order) */ export const valueIsTreeRank = (value: string | undefined): boolean => - value?.startsWith(schema.treeSymbol) || false; + value?.startsWith(schema.treeRankSymbol) ?? false; /** * Returns index from a formatted -to-many index value (e.x #1 => 1) @@ -40,6 +51,14 @@ export const valueIsTreeRank = (value: string | undefined): boolean => export const getNumberFromToManyIndex = (value: string): number => Number(value.slice(schema.referenceSymbol.length)); +/** + * Returns tree definition name from a complete tree definition name + * (e.x %Taxonomy => Taxonomy) + * Opposite of formatTreeDefinition + */ +export const getNameFromTreeDefinitionName = (value: string): string => + value.slice(schema.treeDefinitionSymbol.length); + /* * BUG: in places where output of this function is displayed to the user, * make sure to use tree rank title instead of name @@ -51,7 +70,7 @@ export const getNumberFromToManyIndex = (value: string): number => * */ export const getNameFromTreeRankName = (value: string): string => - value.slice(schema.treeSymbol.length); + value.slice(schema.treeRankSymbol.length); /** * Returns a formatted -to-many index from an index (e.x 1 => #1) @@ -61,7 +80,7 @@ export const formatToManyIndex = (index: number): string => `${schema.referenceSymbol}${index}`; // Meta fields -export const anyTreeRank = `${schema.fieldPartSeparator}any`; +export const anyTreeRank = `${schema.fieldPartSeparator}any` as const; export const formattedEntry = `${schema.fieldPartSeparator}formatted`; /** @@ -75,6 +94,14 @@ export const formattedEntry = `${schema.fieldPartSeparator}formatted`; */ export const emptyMapping = '0'; +/** + * Returns a complete tree definition name from a tree definition name + * (e.x Taxononomy => %Taxonomy) + * Opposite of getNameFromTreeDefinitionName + */ +export const formatTreeDefinition = (definitionName: string): string => + `${schema.treeDefinitionSymbol}${definitionName}`; + /** * Returns a complete tree rank name from a tree rank name * (e.x Kingdom => $Kingdom) @@ -82,7 +109,7 @@ export const emptyMapping = '0'; * */ export const formatTreeRank = (rankName: string): string => - `${schema.treeSymbol}${rankName}`; + `${schema.treeRankSymbol}${rankName}`; // Match fields names like startDate_fullDate, but not _formatted export const valueIsPartialField = (value: string): boolean => @@ -160,7 +187,7 @@ export const getGenericMappingPath = (mappingPath: MappingPath): MappingPath => mappingPath.filter( (mappingPathPart) => !valueIsToManyIndex(mappingPathPart) && - !valueIsTreeRank(mappingPathPart) && + !valueIsTreeMeta(mappingPathPart) && mappingPathPart !== formattedEntry && mappingPathPart !== emptyMapping ); diff --git a/specifyweb/frontend/js_src/lib/components/WbPlanView/navigator.ts b/specifyweb/frontend/js_src/lib/components/WbPlanView/navigator.ts index 341329c0dab..d189273949b 100644 --- a/specifyweb/frontend/js_src/lib/components/WbPlanView/navigator.ts +++ b/specifyweb/frontend/js_src/lib/components/WbPlanView/navigator.ts @@ -10,16 +10,11 @@ import { queryText } from '../../localization/query'; import type { IR, RA, WritableArray } from '../../utils/types'; import { defined, filterArray } from '../../utils/types'; import { dateParts } from '../Atoms/Internationalization'; -import type { - AnyTree, - FilterTablesByEndsWith, - SerializedResource, -} from '../DataModel/helperTypes'; +import type { AnyTree } from '../DataModel/helperTypes'; import type { Relationship } from '../DataModel/specifyField'; import type { SpecifyTable } from '../DataModel/specifyTable'; import { getFrontEndOnlyFields, strictGetTable } from '../DataModel/tables'; import type { Tables } from '../DataModel/types'; -import type { TreeInformation } from '../InitialContext/treeRanks'; import { getTreeDefinitions, isTreeTable } from '../InitialContext/treeRanks'; import { hasTablePermission, hasTreeAccess } from '../Permissions/helpers'; import type { CustomSelectSubtype } from './CustomSelectElement'; @@ -34,13 +29,17 @@ import { formatPartialField, formattedEntry, formatToManyIndex, + formatTreeDefinition, formatTreeRank, getGenericMappingPath, + getNameFromTreeDefinitionName, getNameFromTreeRankName, parsePartialField, relationshipIsToMany, valueIsPartialField, valueIsToManyIndex, + valueIsTreeDefinition, + valueIsTreeMeta, valueIsTreeRank, } from './mappingHelpers'; import { getMaxToManyIndex, isCircularRelationship } from './modelHelpers'; @@ -69,9 +68,14 @@ type NavigationCallbacks = { readonly handleToManyChildren: ( callbackPayload: Readonly ) => void; + readonly handleTreeDefinitions: ( + callbackPayload: Readonly + ) => void; // Handles tree ranks children readonly handleTreeRanks: ( - callbackPayload: Readonly + callbackPayload: Readonly & { + readonly definitionName: string; + } ) => void; // Handles field and relationships readonly handleSimpleFields: ( @@ -117,10 +121,10 @@ function navigator({ const childrenAreToManyElements = relationshipIsToMany(parentRelationship) && !valueIsToManyIndex(parentPartName) && - !valueIsTreeRank(parentPartName); + !valueIsTreeMeta(parentPartName); const childrenAreRanks = - isTreeTable(table.name) && !valueIsTreeRank(parentPartName); + isTreeTable(table.name) && !valueIsTreeMeta(parentPartName); const callbackPayload = { table, @@ -129,11 +133,26 @@ function navigator({ if (childrenAreToManyElements) callbacks.handleToManyChildren(callbackPayload); - else if (childrenAreRanks) callbacks.handleTreeRanks(callbackPayload); + else if (childrenAreRanks) { + const definitions = getTreeDefinitions(table.name, 'all'); + + if (definitions.length > 1) + callbacks.handleTreeDefinitions(callbackPayload); + else + callbacks.handleTreeRanks({ + ...callbackPayload, + definitionName: definitions[0].definition.name ?? anyTreeRank, + }); + } else if (valueIsTreeDefinition(parentPartName)) + callbacks.handleTreeRanks({ + ...callbackPayload, + definitionName: getNameFromTreeDefinitionName(parentPartName), + }); else callbacks.handleSimpleFields(callbackPayload); const isSpecial = - valueIsToManyIndex(next.partName) || valueIsTreeRank(next.partName); + valueIsToManyIndex(next.partName) || valueIsTreeMeta(next.partName); + const nextField = isSpecial ? parentRelationship : table.getField(next.fieldName); @@ -199,8 +218,6 @@ export function getMappingLineData({ showHiddenFields = false, mustMatchPreferences = {}, spec, - taxonType, - treeDefinitions, }: { readonly baseTableName: keyof Tables; // The mapping path @@ -215,10 +232,6 @@ export function getMappingLineData({ readonly showHiddenFields?: boolean; readonly mustMatchPreferences?: IR; readonly spec: NavigatorSpec; - readonly taxonType?: string; - readonly treeDefinitions: - | readonly SerializedResource>[] - | undefined; }): RA { if ( process.env.NODE_ENV !== 'production' && @@ -270,24 +283,7 @@ export function getMappingLineData({ customSelectSubtype, defaultValue: internalState.defaultValue, selectLabel: table.label, - fieldsData: Object.fromEntries( - filterArray(fieldsData) - .filter((field) => { - if (table.name === 'Taxon' && mappingPath[0] === '0') { - return treeDefinitions?.map((definition) => definition.name); - } - /* - * Else if ( - * internalState.mappingLineData.length === 0 && - * taxonType !== undefined - * ) { - * return field[1].tableTreeDefName === taxonType; - * } - */ - return true; - }) - .map(([rawKey, fieldData]) => [rawKey, fieldData]) - ), + fieldsData: Object.fromEntries(filterArray(fieldsData)), tableName: table.name, }); @@ -313,7 +309,7 @@ export function getMappingLineData({ return { partName: nextPart, fieldName: - valueIsTreeRank(nextPart) || valueIsToManyIndex(nextPart) + valueIsTreeMeta(nextPart) || valueIsToManyIndex(nextPart) ? valueIsToManyIndex(mappingPath[internalState.position - 1]) ? mappingPath[internalState.position - 2] : mappingPath[internalState.position - 1] @@ -361,27 +357,67 @@ export function getMappingLineData({ ); }, - handleTreeRanks({ table }) { + handleTreeDefinitions({ table }) { + const defaultValue = getNameFromTreeDefinitionName( + internalState.defaultValue + ); + + commitInstanceData( + 'tree', + table, + generateFieldData === 'none' || + !hasTreeAccess(table.name as AnyTree['tableName'], 'read') + ? [] + : [ + spec.includeAnyTreeDefinition && + (generateFieldData === 'all' || + internalState.defaultValue === + formatTreeDefinition(anyTreeRank)) + ? [ + formatTreeDefinition(anyTreeRank), + { + optionLabel: queryText.anyTree(), + isRelationship: true, + isDefault: + internalState.defaultValue === + formatTreeDefinition(anyTreeRank), + isEnabled: true, + tableName: table.name, + }, + ] + : undefined, + ...(spec.includeSpecificTreeRanks + ? getTreeDefinitions( + table.name as AnyTree['tableName'], + 'all' + ).map(({ definition: { name } }) => + name === defaultValue || generateFieldData === 'all' + ? ([ + formatTreeDefinition(name), + { + optionLabel: name, + isRelationship: true, + isDefault: name === defaultValue, + tableName: table.name, + tableTreeDefName: name, + }, + ] as const) + : undefined + ) + : []), + ] + ); + }, + + handleTreeRanks({ table, definitionName }) { const defaultValue = getNameFromTreeRankName(internalState.defaultValue); - const fieldsTreeData: readonly ( - | readonly [string, HtmlGeneratorFieldData] - | undefined - )[] = + commitInstanceData( + 'tree', + table, generateFieldData === 'none' || - !hasTreeAccess(table.name as 'Geography', 'read') + !hasTreeAccess(table.name as AnyTree['tableName'], 'read') ? [] - : table.name === 'Taxon' && mappingPath[0] === '0' && treeDefinitions - ? treeDefinitions.map((treeDefinition) => [ - treeDefinition.name, - { - optionLabel: treeDefinition.name, - isRelationship: true, - isDefault: false, - tableName: 'Taxon', - tableTreeDefName: treeDefinition.name, - }, - ]) : [ spec.includeAnyTreeRank && (generateFieldData === 'all' || @@ -400,30 +436,32 @@ export function getMappingLineData({ ] : undefined, ...(spec.includeSpecificTreeRanks - ? getTreeDefinitions( - table.name as AnyTree['tableName'], - 'all' - ).flatMap(({ definition, ranks }) => - // Exclude the root rank for each tree - ranks.slice(1).map(({ name, title }) => - name === defaultValue || generateFieldData === 'all' - ? ([ - formatTreeRank(name), - { - optionLabel: title ?? name, - isRelationship: true, - isDefault: name === defaultValue, - tableName: table.name, - tableTreeDefName: definition.name, - }, - ] as const) - : undefined + ? getTreeDefinitions(table.name as AnyTree['tableName'], 'all') + .filter( + ({ definition: { name } }) => + definitionName === name || + definitionName === anyTreeRank + ) + .flatMap(({ definition, ranks }) => + // Exclude the root rank for each tree + ranks.slice(1).map(({ name, title }) => + name === defaultValue || generateFieldData === 'all' + ? ([ + formatTreeRank(name), + { + optionLabel: title ?? name, + isRelationship: true, + isDefault: name === defaultValue, + tableName: table.name, + tableTreeDefName: definition.name, + }, + ] as const) + : undefined + ) ) - ) : []), - ]; - - commitInstanceData('tree', table, fieldsTreeData); + ] + ); }, handleSimpleFields({ table, parentRelationship }) { @@ -636,7 +674,9 @@ export function getMappingLineData({ : internalState.mappingLineData.filter( ({ customSelectSubtype }) => customSelectSubtype !== 'toMany' ); - return spec.includeAnyTreeRank || spec.includeSpecificTreeRanks + return spec.includeAnyTreeRank || + spec.includeAnyTreeDefinition || + spec.includeSpecificTreeRanks ? filtered : filtered.filter( ({ customSelectSubtype }) => customSelectSubtype !== 'tree' diff --git a/specifyweb/frontend/js_src/lib/components/WbPlanView/navigatorSpecs.ts b/specifyweb/frontend/js_src/lib/components/WbPlanView/navigatorSpecs.ts index 643a974ab0d..2aaa30f6457 100644 --- a/specifyweb/frontend/js_src/lib/components/WbPlanView/navigatorSpecs.ts +++ b/specifyweb/frontend/js_src/lib/components/WbPlanView/navigatorSpecs.ts @@ -14,6 +14,7 @@ export type NavigatorSpec = { // Whether no restrictions mode is enabled readonly isNoRestrictions: () => boolean; readonly includeToManyReferenceNumbers: boolean; + readonly includeAnyTreeDefinition: boolean; readonly includeSpecificTreeRanks: boolean; readonly includeAnyTreeRank: boolean; readonly includeFormattedAggregated: boolean; @@ -41,6 +42,7 @@ const wbPlanView: NavigatorSpec = { userPreferences.get('workBench', 'wbPlanView', 'noRestrictionsMode'), includeToManyReferenceNumbers: true, includeSpecificTreeRanks: true, + includeAnyTreeDefinition: false, includeAnyTreeRank: false, includeFormattedAggregated: false, includeRootFormattedAggregated: false, @@ -81,6 +83,7 @@ const queryBuilder: NavigatorSpec = { isNoRestrictions: () => userPreferences.get('queryBuilder', 'general', 'noRestrictionsMode'), includeToManyReferenceNumbers: false, + includeAnyTreeDefinition: true, includeSpecificTreeRanks: true, includeAnyTreeRank: true, includeFormattedAggregated: true, @@ -111,6 +114,7 @@ const formatterEditor: NavigatorSpec = { includeReadOnly: true, isNoRestrictions: () => true, includeToManyReferenceNumbers: false, + includeAnyTreeDefinition: false, includeSpecificTreeRanks: false, includeAnyTreeRank: false, includeFormattedAggregated: true, @@ -133,6 +137,7 @@ const permissive: NavigatorSpec = { includeReadOnly: true, isNoRestrictions: () => true, includeToManyReferenceNumbers: true, + includeAnyTreeDefinition: true, includeSpecificTreeRanks: true, includeAnyTreeRank: true, includeFormattedAggregated: true, diff --git a/specifyweb/frontend/js_src/lib/components/WbPlanView/uploadPlanBuilder.ts b/specifyweb/frontend/js_src/lib/components/WbPlanView/uploadPlanBuilder.ts index ede50bf503d..1d820221071 100644 --- a/specifyweb/frontend/js_src/lib/components/WbPlanView/uploadPlanBuilder.ts +++ b/specifyweb/frontend/js_src/lib/components/WbPlanView/uploadPlanBuilder.ts @@ -38,7 +38,6 @@ const toTreeRecordRanks = ( ]) ); -// TODO: need to update this to include tree def identifier const toTreeRecordVariety = (lines: RA): TreeRecord => ({ ranks: Object.fromEntries( indexMappings(lines).map(([fullRankName, rankMappedFields]) => [ @@ -138,8 +137,7 @@ const toUploadable = ( export const uploadPlanBuilder = ( baseTableName: keyof Tables, lines: RA, - mustMatchPreferences: RR, - taxonTreeId: number | undefined + mustMatchPreferences: RR ): UploadPlan => ({ baseTableName: toLowerCase(baseTableName), uploadable: toUploadable( @@ -150,7 +148,6 @@ export const uploadPlanBuilder = ( .map(([tableName]) => tableName), true ), - taxonTreeId, }); const indexMappings = ( diff --git a/specifyweb/frontend/js_src/lib/components/WbPlanView/uploadPlanParser.ts b/specifyweb/frontend/js_src/lib/components/WbPlanView/uploadPlanParser.ts index 4fc801073c7..3a9463ca621 100644 --- a/specifyweb/frontend/js_src/lib/components/WbPlanView/uploadPlanParser.ts +++ b/specifyweb/frontend/js_src/lib/components/WbPlanView/uploadPlanParser.ts @@ -46,7 +46,6 @@ export type Uploadable = TreeRecordVariety | UploadTableVariety; export type UploadPlan = { readonly baseTableName: Lowercase; readonly uploadable: Uploadable; - readonly taxonTreeId: number | undefined; }; const parseColumnOptions = (matchingOptions: ColumnOptions): ColumnOptions => ({ diff --git a/specifyweb/frontend/js_src/lib/localization/query.ts b/specifyweb/frontend/js_src/lib/localization/query.ts index 3e6785b534f..dc727eb393e 100644 --- a/specifyweb/frontend/js_src/lib/localization/query.ts +++ b/specifyweb/frontend/js_src/lib/localization/query.ts @@ -343,6 +343,9 @@ export const queryText = createDictionary({ 'uk-ua': '(будь-який ранг)', 'de-ch': '(jeder Rang)', }, + anyTree: { + 'en-us': '(any tree)', + }, moveUp: { comment: 'As in move it up', 'en-us': 'Move Up', diff --git a/specifyweb/frontend/js_src/lib/localization/workbench.ts b/specifyweb/frontend/js_src/lib/localization/workbench.ts index 7fd6148ad5e..133aaac36c6 100644 --- a/specifyweb/frontend/js_src/lib/localization/workbench.ts +++ b/specifyweb/frontend/js_src/lib/localization/workbench.ts @@ -1584,7 +1584,4 @@ export const wbText = createDictionary({ 'fr-fr': '{node:string} (dans {parent:string})', 'uk-ua': '{node:string} (у {parent:string})', }, - selectTree: { - 'en-us': 'Select Tree Type', - }, } as const); From 4f657efcb6011b64c5ca9615c1913dc012776ab9 Mon Sep 17 00:00:00 2001 From: melton-jason Date: Wed, 28 Aug 2024 10:00:51 -0500 Subject: [PATCH 31/78] Disable specific tree interface for all components save WB --- .../components/WbPlanView/LineComponents.tsx | 1 - .../WbPlanView/__tests__/navigator.test.ts | 20 ------------------- .../lib/components/WbPlanView/navigator.ts | 15 ++++++++------ .../components/WbPlanView/navigatorSpecs.ts | 7 +++++++ 4 files changed, 16 insertions(+), 27 deletions(-) diff --git a/specifyweb/frontend/js_src/lib/components/WbPlanView/LineComponents.tsx b/specifyweb/frontend/js_src/lib/components/WbPlanView/LineComponents.tsx index df5aceaf43f..433a4761b77 100644 --- a/specifyweb/frontend/js_src/lib/components/WbPlanView/LineComponents.tsx +++ b/specifyweb/frontend/js_src/lib/components/WbPlanView/LineComponents.tsx @@ -37,7 +37,6 @@ export type HtmlGeneratorFieldData = { readonly isDefault?: boolean; readonly isRelationship?: boolean; readonly tableName?: keyof Tables; - readonly tableTreeDefName?: string; }; type MappingLineBaseProps = { diff --git a/specifyweb/frontend/js_src/lib/components/WbPlanView/__tests__/navigator.test.ts b/specifyweb/frontend/js_src/lib/components/WbPlanView/__tests__/navigator.test.ts index 7ebc8389786..cec41085ea3 100644 --- a/specifyweb/frontend/js_src/lib/components/WbPlanView/__tests__/navigator.test.ts +++ b/specifyweb/frontend/js_src/lib/components/WbPlanView/__tests__/navigator.test.ts @@ -314,70 +314,60 @@ theories(getMappingLineData, [ isRelationship: true, isDefault: false, tableName: 'Taxon', - tableTreeDefName: 'Taxonomy', }, $Phylum: { optionLabel: 'Phylum', isRelationship: true, isDefault: false, tableName: 'Taxon', - tableTreeDefName: 'Taxonomy', }, $Class: { optionLabel: 'Class', isRelationship: true, isDefault: false, tableName: 'Taxon', - tableTreeDefName: 'Taxonomy', }, $Order: { optionLabel: 'Order', isRelationship: true, isDefault: false, tableName: 'Taxon', - tableTreeDefName: 'Taxonomy', }, $Family: { optionLabel: 'Family', isRelationship: true, isDefault: true, tableName: 'Taxon', - tableTreeDefName: 'Taxonomy', }, $Subfamily: { optionLabel: 'Subfamily', isRelationship: true, isDefault: false, tableName: 'Taxon', - tableTreeDefName: 'Taxonomy', }, $Genus: { optionLabel: 'Genus', isRelationship: true, isDefault: false, tableName: 'Taxon', - tableTreeDefName: 'Taxonomy', }, $Subgenus: { optionLabel: 'Subgenus', isRelationship: true, isDefault: false, tableName: 'Taxon', - tableTreeDefName: 'Taxonomy', }, $Species: { optionLabel: 'Species', isRelationship: true, isDefault: false, tableName: 'Taxon', - tableTreeDefName: 'Taxonomy', }, $Subspecies: { optionLabel: 'Subspecies', isRelationship: true, isDefault: false, tableName: 'Taxon', - tableTreeDefName: 'Taxonomy', }, }, tableName: 'Taxon', @@ -870,70 +860,60 @@ theories(getMappingLineData, [ isRelationship: true, optionLabel: 'Class', tableName: 'Taxon', - tableTreeDefName: 'Taxonomy', }, $Family: { isDefault: true, isRelationship: true, optionLabel: 'Family', tableName: 'Taxon', - tableTreeDefName: 'Taxonomy', }, $Genus: { isDefault: false, isRelationship: true, optionLabel: 'Genus', tableName: 'Taxon', - tableTreeDefName: 'Taxonomy', }, $Kingdom: { isDefault: false, isRelationship: true, optionLabel: 'Kingdom', tableName: 'Taxon', - tableTreeDefName: 'Taxonomy', }, $Order: { isDefault: false, isRelationship: true, optionLabel: 'Order', tableName: 'Taxon', - tableTreeDefName: 'Taxonomy', }, $Phylum: { isDefault: false, isRelationship: true, optionLabel: 'Phylum', tableName: 'Taxon', - tableTreeDefName: 'Taxonomy', }, $Species: { isDefault: false, isRelationship: true, optionLabel: 'Species', tableName: 'Taxon', - tableTreeDefName: 'Taxonomy', }, $Subfamily: { isDefault: false, isRelationship: true, optionLabel: 'Subfamily', tableName: 'Taxon', - tableTreeDefName: 'Taxonomy', }, $Subgenus: { isDefault: false, isRelationship: true, optionLabel: 'Subgenus', tableName: 'Taxon', - tableTreeDefName: 'Taxonomy', }, $Subspecies: { isDefault: false, isRelationship: true, optionLabel: 'Subspecies', tableName: 'Taxon', - tableTreeDefName: 'Taxonomy', }, }, selectLabel: localized('Taxon'), diff --git a/specifyweb/frontend/js_src/lib/components/WbPlanView/navigator.ts b/specifyweb/frontend/js_src/lib/components/WbPlanView/navigator.ts index d189273949b..d551d1e996e 100644 --- a/specifyweb/frontend/js_src/lib/components/WbPlanView/navigator.ts +++ b/specifyweb/frontend/js_src/lib/components/WbPlanView/navigator.ts @@ -40,7 +40,6 @@ import { valueIsToManyIndex, valueIsTreeDefinition, valueIsTreeMeta, - valueIsTreeRank, } from './mappingHelpers'; import { getMaxToManyIndex, isCircularRelationship } from './modelHelpers'; import type { NavigatorSpec } from './navigatorSpecs'; @@ -93,10 +92,12 @@ type NavigationCallbacks = { * fields from multiple relationships at once) */ function navigator({ + spec, callbacks, recursivePayload, baseTableName, }: { + readonly spec: NavigatorSpec; // Callbacks can be modified depending on the need to make navigator versatile readonly callbacks: NavigationCallbacks; // Used internally to make navigator call itself multiple times @@ -136,12 +137,14 @@ function navigator({ else if (childrenAreRanks) { const definitions = getTreeDefinitions(table.name, 'all'); - if (definitions.length > 1) + if (definitions.length > 1 && spec.useSpecificTreeInterface) callbacks.handleTreeDefinitions(callbackPayload); else callbacks.handleTreeRanks({ ...callbackPayload, - definitionName: definitions[0].definition.name ?? anyTreeRank, + definitionName: spec.useSpecificTreeInterface + ? definitions[0].definition.name ?? anyTreeRank + : anyTreeRank, }); } else if (valueIsTreeDefinition(parentPartName)) callbacks.handleTreeRanks({ @@ -165,6 +168,7 @@ function navigator({ if (typeof nextTable === 'object' && nextField?.isRelationship !== false) navigator({ + spec, callbacks, recursivePayload: { table: nextTable, @@ -399,7 +403,6 @@ export function getMappingLineData({ isRelationship: true, isDefault: name === defaultValue, tableName: table.name, - tableTreeDefName: name, }, ] as const) : undefined @@ -442,7 +445,7 @@ export function getMappingLineData({ definitionName === name || definitionName === anyTreeRank ) - .flatMap(({ definition, ranks }) => + .flatMap(({ ranks }) => // Exclude the root rank for each tree ranks.slice(1).map(({ name, title }) => name === defaultValue || generateFieldData === 'all' @@ -453,7 +456,6 @@ export function getMappingLineData({ isRelationship: true, isDefault: name === defaultValue, tableName: table.name, - tableTreeDefName: definition.name, }, ] as const) : undefined @@ -665,6 +667,7 @@ export function getMappingLineData({ }; navigator({ + spec, callbacks, baseTableName, }); diff --git a/specifyweb/frontend/js_src/lib/components/WbPlanView/navigatorSpecs.ts b/specifyweb/frontend/js_src/lib/components/WbPlanView/navigatorSpecs.ts index 2aaa30f6457..3619bb9e853 100644 --- a/specifyweb/frontend/js_src/lib/components/WbPlanView/navigatorSpecs.ts +++ b/specifyweb/frontend/js_src/lib/components/WbPlanView/navigatorSpecs.ts @@ -14,6 +14,8 @@ export type NavigatorSpec = { // Whether no restrictions mode is enabled readonly isNoRestrictions: () => boolean; readonly includeToManyReferenceNumbers: boolean; + // REFACTOR: remove this attribute + readonly useSpecificTreeInterface: boolean; readonly includeAnyTreeDefinition: boolean; readonly includeSpecificTreeRanks: boolean; readonly includeAnyTreeRank: boolean; @@ -41,6 +43,7 @@ const wbPlanView: NavigatorSpec = { isNoRestrictions: () => userPreferences.get('workBench', 'wbPlanView', 'noRestrictionsMode'), includeToManyReferenceNumbers: true, + useSpecificTreeInterface: true, includeSpecificTreeRanks: true, includeAnyTreeDefinition: false, includeAnyTreeRank: false, @@ -83,6 +86,8 @@ const queryBuilder: NavigatorSpec = { isNoRestrictions: () => userPreferences.get('queryBuilder', 'general', 'noRestrictionsMode'), includeToManyReferenceNumbers: false, + // REFACTOR: see https://github.com/specify/specify7/pull/5251 + useSpecificTreeInterface: false, includeAnyTreeDefinition: true, includeSpecificTreeRanks: true, includeAnyTreeRank: true, @@ -114,6 +119,7 @@ const formatterEditor: NavigatorSpec = { includeReadOnly: true, isNoRestrictions: () => true, includeToManyReferenceNumbers: false, + useSpecificTreeInterface: false, includeAnyTreeDefinition: false, includeSpecificTreeRanks: false, includeAnyTreeRank: false, @@ -137,6 +143,7 @@ const permissive: NavigatorSpec = { includeReadOnly: true, isNoRestrictions: () => true, includeToManyReferenceNumbers: true, + useSpecificTreeInterface: true, includeAnyTreeDefinition: true, includeSpecificTreeRanks: true, includeAnyTreeRank: true, From 5ad335851f40aaf74c63232609310705be77eb06 Mon Sep 17 00:00:00 2001 From: Sharad S Date: Wed, 28 Aug 2024 14:57:55 -0400 Subject: [PATCH 32/78] Pass tree definition id in API request --- .../WbPlanView/uploadPlanBuilder.ts | 57 ++++++++++++++----- 1 file changed, 44 insertions(+), 13 deletions(-) diff --git a/specifyweb/frontend/js_src/lib/components/WbPlanView/uploadPlanBuilder.ts b/specifyweb/frontend/js_src/lib/components/WbPlanView/uploadPlanBuilder.ts index 1d820221071..db08f892c89 100644 --- a/specifyweb/frontend/js_src/lib/components/WbPlanView/uploadPlanBuilder.ts +++ b/specifyweb/frontend/js_src/lib/components/WbPlanView/uploadPlanBuilder.ts @@ -3,9 +3,14 @@ import { group, removeKey, split, toLowerCase } from '../../utils/utils'; import type { SpecifyTable } from '../DataModel/specifyTable'; import { strictGetTable } from '../DataModel/tables'; import type { Tables } from '../DataModel/types'; -import { isTreeTable } from '../InitialContext/treeRanks'; +import { getTreeDefinitions, isTreeTable } from '../InitialContext/treeRanks'; import { defaultColumnOptions } from './linesGetter'; -import type { SplitMappingPath } from './mappingHelpers'; +import { + getNameFromTreeDefinitionName, + SplitMappingPath, + valueIsTreeDefinition, + valueIsTreeRank, +} from './mappingHelpers'; import { getNameFromTreeRankName, valueIsToManyIndex } from './mappingHelpers'; import type { ColumnDefinition, @@ -33,21 +38,47 @@ const toTreeRecordRanks = ( ): IR => Object.fromEntries( rankMappedFields.map(({ mappingPath, headerName, columnOptions }) => [ - mappingPath[0].toLowerCase(), + // Use the last element of the mapping path to get the field name + // e.g: mappingPath when a tree is specified: ['$Kingdom', 'name'] vs when no tree is specified: ['name'] + mappingPath[mappingPath.length - 1].toLowerCase(), toColumnOptions(headerName, columnOptions), ]) ); -const toTreeRecordVariety = (lines: RA): TreeRecord => ({ - ranks: Object.fromEntries( - indexMappings(lines).map(([fullRankName, rankMappedFields]) => [ - getNameFromTreeRankName(fullRankName), - { - treeNodeCols: toTreeRecordRanks(rankMappedFields), - }, - ]) - ), -}); +const toTreeRecordVariety = (lines: RA): TreeRecord => { + const treeDefinitions = getTreeDefinitions('Taxon', 'all'); + + return { + ranks: Object.fromEntries( + indexMappings(lines).map(([fullName, rankMappedFields]) => { + const rankName = valueIsTreeRank(fullName) + ? getNameFromTreeRankName(fullName) + : getNameFromTreeRankName(rankMappedFields[0].mappingPath[0]); + + const treeName = valueIsTreeDefinition(fullName) + ? getNameFromTreeDefinitionName(fullName) + : undefined; + + const treeId = + treeName !== undefined + ? treeDefinitions.find( + ({ definition }) => definition.name === treeName + )?.definition.id + : treeDefinitions.find(({ ranks }) => + ranks.find((r) => r.name === rankName) + )?.definition.id; + + return [ + rankName, + { + treeNodeCols: toTreeRecordRanks(rankMappedFields), + treeId, + }, + ]; + }) + ), + }; +}; function toUploadTable( table: SpecifyTable, From 004dda41491f96c45f81cdc5f78aa4fc0d102ea7 Mon Sep 17 00:00:00 2001 From: Sharad S Date: Wed, 28 Aug 2024 15:13:51 -0400 Subject: [PATCH 33/78] Fix tests --- .../DataModel/__tests__/schemaBase.test.ts | 3 ++- .../components/Syncer/__tests__/index.test.ts | 4 ++-- .../lib/tests/fixtures/uploadplan.1.json | 18 ++++++++++++------ 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/specifyweb/frontend/js_src/lib/components/DataModel/__tests__/schemaBase.test.ts b/specifyweb/frontend/js_src/lib/components/DataModel/__tests__/schemaBase.test.ts index 8e16373ae35..3aea215586a 100644 --- a/specifyweb/frontend/js_src/lib/components/DataModel/__tests__/schemaBase.test.ts +++ b/specifyweb/frontend/js_src/lib/components/DataModel/__tests__/schemaBase.test.ts @@ -24,5 +24,6 @@ test('domain data is fetched and parsed correctly', async () => ], paleoContextChildTable: 'collectionobject', referenceSymbol: '#', - treeSymbol: '$', + treeRankSymbol: '$', + treeDefinitionSymbol: '%' })); diff --git a/specifyweb/frontend/js_src/lib/components/Syncer/__tests__/index.test.ts b/specifyweb/frontend/js_src/lib/components/Syncer/__tests__/index.test.ts index 89b362c28c0..bb7abf22a9c 100644 --- a/specifyweb/frontend/js_src/lib/components/Syncer/__tests__/index.test.ts +++ b/specifyweb/frontend/js_src/lib/components/Syncer/__tests__/index.test.ts @@ -94,8 +94,8 @@ test('Editing Data Object Formatter', () => { - - + + " `); diff --git a/specifyweb/frontend/js_src/lib/tests/fixtures/uploadplan.1.json b/specifyweb/frontend/js_src/lib/tests/fixtures/uploadplan.1.json index f0cc682997a..a639bc1327d 100644 --- a/specifyweb/frontend/js_src/lib/tests/fixtures/uploadplan.1.json +++ b/specifyweb/frontend/js_src/lib/tests/fixtures/uploadplan.1.json @@ -283,34 +283,40 @@ "default": null, "column": "Class" } - } + }, + "treeId": 1 }, "Family": { "treeNodeCols": { "name": "Family" - } + }, + "treeId": 1 }, "Genus": { "treeNodeCols": { "name": "Genus" - } + }, + "treeId": 1 }, "Subgenus": { "treeNodeCols": { "name": "Subgenus" - } + }, + "treeId": 1 }, "Species": { "treeNodeCols": { "name": "Species", "author": "Species Author" - } + }, + "treeId": 1 }, "Subspecies": { "treeNodeCols": { "name": "Subspecies", "author": "Subspecies Author" - } + }, + "treeId": 1 } } } From 47c9fdf88bd364888300d646981b2ed1f6314ff2 Mon Sep 17 00:00:00 2001 From: Sharad S Date: Wed, 28 Aug 2024 19:17:37 +0000 Subject: [PATCH 34/78] Lint code with ESLint and Prettier Triggered by 004dda41491f96c45f81cdc5f78aa4fc0d102ea7 on branch refs/heads/issue-4980 --- .../DataModel/__tests__/schemaBase.test.ts | 2 +- .../WbPlanView/uploadPlanBuilder.ts | 20 ++++++++++--------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/specifyweb/frontend/js_src/lib/components/DataModel/__tests__/schemaBase.test.ts b/specifyweb/frontend/js_src/lib/components/DataModel/__tests__/schemaBase.test.ts index 3aea215586a..78524afc219 100644 --- a/specifyweb/frontend/js_src/lib/components/DataModel/__tests__/schemaBase.test.ts +++ b/specifyweb/frontend/js_src/lib/components/DataModel/__tests__/schemaBase.test.ts @@ -25,5 +25,5 @@ test('domain data is fetched and parsed correctly', async () => paleoContextChildTable: 'collectionobject', referenceSymbol: '#', treeRankSymbol: '$', - treeDefinitionSymbol: '%' + treeDefinitionSymbol: '%', })); diff --git a/specifyweb/frontend/js_src/lib/components/WbPlanView/uploadPlanBuilder.ts b/specifyweb/frontend/js_src/lib/components/WbPlanView/uploadPlanBuilder.ts index db08f892c89..27b17cd1240 100644 --- a/specifyweb/frontend/js_src/lib/components/WbPlanView/uploadPlanBuilder.ts +++ b/specifyweb/frontend/js_src/lib/components/WbPlanView/uploadPlanBuilder.ts @@ -5,9 +5,9 @@ import { strictGetTable } from '../DataModel/tables'; import type { Tables } from '../DataModel/types'; import { getTreeDefinitions, isTreeTable } from '../InitialContext/treeRanks'; import { defaultColumnOptions } from './linesGetter'; +import type { SplitMappingPath } from './mappingHelpers'; import { getNameFromTreeDefinitionName, - SplitMappingPath, valueIsTreeDefinition, valueIsTreeRank, } from './mappingHelpers'; @@ -38,9 +38,11 @@ const toTreeRecordRanks = ( ): IR => Object.fromEntries( rankMappedFields.map(({ mappingPath, headerName, columnOptions }) => [ - // Use the last element of the mapping path to get the field name - // e.g: mappingPath when a tree is specified: ['$Kingdom', 'name'] vs when no tree is specified: ['name'] - mappingPath[mappingPath.length - 1].toLowerCase(), + /* + * Use the last element of the mapping path to get the field name + * e.g: mappingPath when a tree is specified: ['$Kingdom', 'name'] vs when no tree is specified: ['name'] + */ + mappingPath.at(-1).toLowerCase(), toColumnOptions(headerName, columnOptions), ]) ); @@ -60,12 +62,12 @@ const toTreeRecordVariety = (lines: RA): TreeRecord => { : undefined; const treeId = - treeName !== undefined - ? treeDefinitions.find( - ({ definition }) => definition.name === treeName - )?.definition.id - : treeDefinitions.find(({ ranks }) => + treeName === undefined + ? treeDefinitions.find(({ ranks }) => ranks.find((r) => r.name === rankName) + )?.definition.id + : treeDefinitions.find( + ({ definition }) => definition.name === treeName )?.definition.id; return [ From 790bacab19884415315ce780c2c3b16b8b7739b4 Mon Sep 17 00:00:00 2001 From: alec_dev Date: Thu, 29 Aug 2024 10:15:35 -0500 Subject: [PATCH 35/78] TreeRank NamedTuple --- specifyweb/specify/tree_utils.py | 2 + specifyweb/workbench/upload/scoping.py | 2 +- specifyweb/workbench/upload/treerecord.py | 99 +++++++++---------- .../workbench/upload/upload_plan_schema.py | 2 +- 4 files changed, 52 insertions(+), 53 deletions(-) diff --git a/specifyweb/specify/tree_utils.py b/specifyweb/specify/tree_utils.py index af442bd6787..ac9cd0783a0 100644 --- a/specifyweb/specify/tree_utils.py +++ b/specifyweb/specify/tree_utils.py @@ -5,6 +5,8 @@ lookup = lambda tree: (tree.lower() + 'treedef') +SPECIFY_TREES = {"taxon", "storage", "geography", "geologictimeperiod", "lithostrat"} + def get_search_filters(collection: spmodels.Collection, tree: str): tree_name = tree.lower() if tree_name == 'storage': diff --git a/specifyweb/workbench/upload/scoping.py b/specifyweb/workbench/upload/scoping.py index 848977c3b9a..9e1f9338491 100644 --- a/specifyweb/workbench/upload/scoping.py +++ b/specifyweb/workbench/upload/scoping.py @@ -222,7 +222,7 @@ def apply_scoping_to_treerecord(tr: TreeRecord, collection) -> ScopedTreeRecord: scoped_ranks: Dict[TreeRankRecord, Dict[str, ExtendedColumnOptions]] = {} for r, cols in tr.ranks.items(): if isinstance(r, str): - r = TreeRank(r, table.name, treedef.id if treedef else None, tr.base_treedef_id).tree_rank_record() + r = TreeRank.create(r, table.name, treedef.id if treedef else None, tr.base_treedef_id).tree_rank_record() scoped_ranks[r] = {} for f, colopts in cols.items(): scoped_ranks[r][f] = extend_columnoptions(colopts, collection, table.name, f) diff --git a/specifyweb/workbench/upload/treerecord.py b/specifyweb/workbench/upload/treerecord.py index bd257e3e84e..c3e3c5b2d0f 100644 --- a/specifyweb/workbench/upload/treerecord.py +++ b/specifyweb/workbench/upload/treerecord.py @@ -10,6 +10,7 @@ from specifyweb.businessrules.exceptions import BusinessRuleException from specifyweb.specify import models +from specifyweb.specify.tree_utils import SPECIFY_TREES from .column_options import ColumnOptions, ExtendedColumnOptions from .parsing import ParseResult, WorkBenchParseFailure, parse_many, filter_and_upload from .upload_result import UploadResult, NullRecord, NoMatch, Matched, \ @@ -19,62 +20,61 @@ logger = logging.getLogger(__name__) -class TreeRank: +class TreeRank(NamedTuple): rank_name: str treedef_id: int tree: str - def __init__( - self, + @staticmethod + def create( rank_name: str, tree: str, treedef_id: Optional[int] = None, base_treedef_id: Optional[int] = None, - ): - self.rank_name = rank_name - self.treedef_id = self._get_treedef_id(rank_name, tree, treedef_id, base_treedef_id) - self.tree = tree.lower() - - def _get_treedef_id( - self, - rank_name: str, - tree: str, - treedef_id: Optional[int], - base_treedef_id: Optional[int], - ) -> int: + ) -> 'TreeRank': tree_model = get_tree_model(tree) - - filter_kwargs = self._build_filter_kwargs(rank_name, treedef_id) - treedefitems = tree_model.objects.filter(**filter_kwargs) - - if not treedefitems.exists() and treedef_id is not None: - filter_kwargs = self._build_filter_kwargs(rank_name) + tree_lower = tree.lower() + + def _build_filter_kwargs(rank_name: str, treedef_id: Optional[int] = None) -> Dict[str, Any]: + filter_kwargs = {'name': rank_name} + if treedef_id is not None: + filter_kwargs['treedef_id'] = treedef_id # type: ignore + return filter_kwargs + + def _filter_by_base_treedef_id(treedefitems, rank_name: str, base_treedef_id: Optional[int]): + if base_treedef_id is None: + raise ValueError(f"Multiple treedefitems found for rank {rank_name}") + treedefitems = treedefitems.filter(treedef_id=base_treedef_id) + if not treedefitems.exists(): + raise ValueError(f"No treedefitems found for rank {rank_name} and base treedef {base_treedef_id}") + if treedefitems.count() > 1: + raise ValueError(f"Multiple treedefitems found for rank {rank_name} and base treedef {base_treedef_id}") + return treedefitems + + def _get_treedef_id( + rank_name: str, + tree: str, + treedef_id: Optional[int], + base_treedef_id: Optional[int], + ) -> int: + filter_kwargs = _build_filter_kwargs(rank_name, treedef_id) treedefitems = tree_model.objects.filter(**filter_kwargs) + + if not treedefitems.exists() and treedef_id is not None: + filter_kwargs = _build_filter_kwargs(rank_name) + treedefitems = tree_model.objects.filter(**filter_kwargs) - if treedefitems.count() > 1: - treedefitems = self._filter_by_base_treedef_id(treedefitems, rank_name, base_treedef_id) - - first_item = treedefitems.first() - if first_item is None: - raise ValueError(f"No treedefitems found for rank {rank_name}") + if treedefitems.count() > 1: + treedefitems = _filter_by_base_treedef_id(treedefitems, rank_name, base_treedef_id) - return first_item.treedef_id + first_item = treedefitems.first() + if first_item is None: + raise ValueError(f"No treedefitems found for rank {rank_name}") - def _build_filter_kwargs(self, rank_name: str, treedef_id: Optional[int] = None) -> Dict[str, Any]: - filter_kwargs = {'name': rank_name} - if treedef_id is not None: - filter_kwargs['treedef_id'] = treedef_id # type: ignore - return filter_kwargs + return first_item.treedef_id - def _filter_by_base_treedef_id(self, treedefitems, rank_name: str, base_treedef_id: Optional[int]): - if base_treedef_id is None: - raise ValueError(f"Multiple treedefitems found for rank {rank_name}") - treedefitems = treedefitems.filter(treedef_id=base_treedef_id) - if not treedefitems.exists(): - raise ValueError(f"No treedefitems found for rank {rank_name} and base treedef {base_treedef_id}") - if treedefitems.count() > 1: - raise ValueError(f"Multiple treedefitems found for rank {rank_name} and base treedef {base_treedef_id}") - return treedefitems + treedef_id = _get_treedef_id(rank_name, tree, treedef_id, base_treedef_id) + return TreeRank(rank_name, treedef_id, tree_lower) def check_rank(self) -> bool: tree_model = get_tree_model(self.tree) @@ -88,9 +88,7 @@ def validate_rank(self) -> None: def tree_rank_record(self) -> 'TreeRankRecord': assert self.rank_name is not None, "Rank name is required" assert self.treedef_id is not None, "Treedef ID is required" - assert self.tree is not None and self.tree.lower() in { - "taxon", "storage", "geography", "geologictimeperiod", "lithostrat" # TODO: Replace with constants - }, "Tree is required" + assert self.tree is not None and self.tree.lower() in SPECIFY_TREES, "Tree is required" return TreeRankRecord(self.rank_name, self.treedef_id) def get_tree_model(tree: str): @@ -107,13 +105,13 @@ def to_json(self) -> Dict: } def to_key(self) -> Tuple[str, int]: - return (self.rank_name, self.treedef_id) # Check this line in unit test + return (self.rank_name, self.treedef_id) def check_rank(self, tree: str) -> bool: - return TreeRank(self.rank_name, tree, self.treedef_id).check_rank() + return TreeRank.create(self.rank_name, tree, self.treedef_id).check_rank() def validate_rank(self, tree) -> None: - TreeRank(self.rank_name, tree, self.treedef_id).validate_rank() + TreeRank.create(self.rank_name, tree, self.treedef_id).validate_rank() class TreeRecord(NamedTuple): name: str @@ -128,11 +126,10 @@ def get_cols(self) -> Set[str]: return {col.column for r in self.ranks.values() for col in r.values() if hasattr(col, 'column')} def to_json(self) -> Dict: - # result: Dict[str, Union[str, int, Dict[str, Any]]] = {"ranks": {}} result = {"ranks": {}} # type: ignore for rank, cols in self.ranks.items(): - rank_key = rank.rank_name if isinstance(rank, TreeRankRecord) else rank # Check this line in unit test + rank_key = rank.rank_name if isinstance(rank, TreeRankRecord) else rank treeNodeCols = {k: v.to_json() if hasattr(v, "to_json") else v for k, v in cols.items()} if len(cols) == 1: @@ -167,7 +164,7 @@ def bind(self, collection, row: Row, uploadingAgentId: Optional[int], auditor: A parseFails: List[WorkBenchParseFailure] = [] for tree_rank_record, cols in self.ranks.items(): nameColumn = cols['name'] - presults, pfails = parse_many(collection, self.name, cols, row) # TODO: fix + presults, pfails = parse_many(collection, self.name, cols, row) parsedFields[tree_rank_record.rank_name] = presults parseFails += pfails filters = {k: v for result in presults for k, v in result.filter_on.items()} diff --git a/specifyweb/workbench/upload/upload_plan_schema.py b/specifyweb/workbench/upload/upload_plan_schema.py index 3c2f4171038..645e3eb71d7 100644 --- a/specifyweb/workbench/upload/upload_plan_schema.py +++ b/specifyweb/workbench/upload/upload_plan_schema.py @@ -320,7 +320,7 @@ def parse_rank(name_or_cols): ranks: Dict[Union[str, TreeRankRecord], Dict[str, ColumnOptions]] = {} for rank, name_or_cols in to_parse['ranks'].items(): rank_name, parsed_cols, treedefid = parse_rank(name_or_cols) - tree_rank_record = TreeRank(rank, table.name, treedefid, base_treedefid).tree_rank_record() + tree_rank_record = TreeRank.create(rank, table.name, treedefid, base_treedefid).tree_rank_record() ranks[tree_rank_record] = parsed_cols # Update the schema with the treeId From ca89ba2cf46fe4afd7cd7d55ee9dd34ba677b9cc Mon Sep 17 00:00:00 2001 From: Sharad S Date: Fri, 30 Aug 2024 13:37:53 -0400 Subject: [PATCH 36/78] Parse treeId in upload plan when creating lines --- .../components/WbPlanView/uploadPlanParser.ts | 21 ++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/specifyweb/frontend/js_src/lib/components/WbPlanView/uploadPlanParser.ts b/specifyweb/frontend/js_src/lib/components/WbPlanView/uploadPlanParser.ts index 3a9463ca621..30924fa7756 100644 --- a/specifyweb/frontend/js_src/lib/components/WbPlanView/uploadPlanParser.ts +++ b/specifyweb/frontend/js_src/lib/components/WbPlanView/uploadPlanParser.ts @@ -3,9 +3,10 @@ import type { SpecifyTable } from '../DataModel/specifyTable'; import { strictGetTable } from '../DataModel/tables'; import type { Tables } from '../DataModel/types'; import { softFail } from '../Errors/Crash'; +import { getTreeDefinitions } from '../InitialContext/treeRanks'; import { defaultColumnOptions } from './linesGetter'; import type { MappingPath } from './Mapper'; -import type { SplitMappingPath } from './mappingHelpers'; +import { formatTreeDefinition, SplitMappingPath } from './mappingHelpers'; import { formatToManyIndex, formatTreeRank } from './mappingHelpers'; export type MatchBehaviors = 'ignoreAlways' | 'ignoreNever' | 'ignoreWhenBlank'; @@ -34,7 +35,10 @@ type UploadTableVariety = | { readonly uploadTable: UploadTable }; export type TreeRecord = { - readonly ranks: IR }>; + readonly ranks: IR< + | string + | { readonly treeNodeCols: IR; readonly treeId?: number } + >; }; type TreeRecordVariety = @@ -70,10 +74,21 @@ const parseTree = ( name: rankData, } : rankData.treeNodeCols, - [...mappingPath, formatTreeRank(rankName)] + [ + ...mappingPath, + ...(typeof rankData === 'object' && typeof rankData.treeId === 'number' + ? [resolveTreeId(rankData.treeId)] + : []), + formatTreeRank(rankName), + ] ) ); +const resolveTreeId = (id: number): string => { + const treeDefinition = getTreeDefinitions('Taxon', id); + return formatTreeDefinition(treeDefinition[0].definition.name); +}; + function parseTreeTypes( table: SpecifyTable, uploadPlan: TreeRecordVariety, From d4af2d600e1d6f56cf996651bae575ad885ca558 Mon Sep 17 00:00:00 2001 From: Sharad S Date: Fri, 30 Aug 2024 13:37:58 -0400 Subject: [PATCH 37/78] Fix typecheck --- .../js_src/lib/components/WbPlanView/uploadPlanBuilder.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specifyweb/frontend/js_src/lib/components/WbPlanView/uploadPlanBuilder.ts b/specifyweb/frontend/js_src/lib/components/WbPlanView/uploadPlanBuilder.ts index 27b17cd1240..b839b5719b1 100644 --- a/specifyweb/frontend/js_src/lib/components/WbPlanView/uploadPlanBuilder.ts +++ b/specifyweb/frontend/js_src/lib/components/WbPlanView/uploadPlanBuilder.ts @@ -42,7 +42,7 @@ const toTreeRecordRanks = ( * Use the last element of the mapping path to get the field name * e.g: mappingPath when a tree is specified: ['$Kingdom', 'name'] vs when no tree is specified: ['name'] */ - mappingPath.at(-1).toLowerCase(), + mappingPath.at(-1)?.toLowerCase(), toColumnOptions(headerName, columnOptions), ]) ); From 9c0b163f8cb6b36850a3cea1b0af142acec3f080 Mon Sep 17 00:00:00 2001 From: Sharad S Date: Fri, 30 Aug 2024 14:27:52 -0400 Subject: [PATCH 38/78] Fix treeId parsing for collections with 1 tree --- .../js_src/lib/components/WbPlanView/uploadPlanParser.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/specifyweb/frontend/js_src/lib/components/WbPlanView/uploadPlanParser.ts b/specifyweb/frontend/js_src/lib/components/WbPlanView/uploadPlanParser.ts index 30924fa7756..15724d9a741 100644 --- a/specifyweb/frontend/js_src/lib/components/WbPlanView/uploadPlanParser.ts +++ b/specifyweb/frontend/js_src/lib/components/WbPlanView/uploadPlanParser.ts @@ -76,7 +76,9 @@ const parseTree = ( : rankData.treeNodeCols, [ ...mappingPath, - ...(typeof rankData === 'object' && typeof rankData.treeId === 'number' + ...(typeof rankData === 'object' && + typeof rankData.treeId === 'number' && + getTreeDefinitions('Taxon', 'all').length > 1 ? [resolveTreeId(rankData.treeId)] : []), formatTreeRank(rankName), From 4a4d58ad963f95f9db509c67100deb90015d8adf Mon Sep 17 00:00:00 2001 From: Sharad S Date: Fri, 30 Aug 2024 18:31:30 +0000 Subject: [PATCH 39/78] Lint code with ESLint and Prettier Triggered by 9c0b163f8cb6b36850a3cea1b0af142acec3f080 on branch refs/heads/issue-4980 --- .../js_src/lib/components/WbPlanView/uploadPlanParser.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/specifyweb/frontend/js_src/lib/components/WbPlanView/uploadPlanParser.ts b/specifyweb/frontend/js_src/lib/components/WbPlanView/uploadPlanParser.ts index 15724d9a741..b1378674013 100644 --- a/specifyweb/frontend/js_src/lib/components/WbPlanView/uploadPlanParser.ts +++ b/specifyweb/frontend/js_src/lib/components/WbPlanView/uploadPlanParser.ts @@ -6,7 +6,8 @@ import { softFail } from '../Errors/Crash'; import { getTreeDefinitions } from '../InitialContext/treeRanks'; import { defaultColumnOptions } from './linesGetter'; import type { MappingPath } from './Mapper'; -import { formatTreeDefinition, SplitMappingPath } from './mappingHelpers'; +import type { SplitMappingPath } from './mappingHelpers'; +import { formatTreeDefinition } from './mappingHelpers'; import { formatToManyIndex, formatTreeRank } from './mappingHelpers'; export type MatchBehaviors = 'ignoreAlways' | 'ignoreNever' | 'ignoreWhenBlank'; From 5ef3bfac65a1377dbc3471a6260bf69c1edf7554 Mon Sep 17 00:00:00 2001 From: alec_dev Date: Tue, 3 Sep 2024 12:50:20 -0500 Subject: [PATCH 40/78] add rescope_tree_from_row in wb tree binding --- specifyweb/workbench/upload/treerecord.py | 56 +++++++++++++++++-- .../workbench/upload/upload_plan_schema.py | 2 +- 2 files changed, 51 insertions(+), 7 deletions(-) diff --git a/specifyweb/workbench/upload/treerecord.py b/specifyweb/workbench/upload/treerecord.py index c3e3c5b2d0f..4db1eb4f051 100644 --- a/specifyweb/workbench/upload/treerecord.py +++ b/specifyweb/workbench/upload/treerecord.py @@ -32,7 +32,7 @@ def create( treedef_id: Optional[int] = None, base_treedef_id: Optional[int] = None, ) -> 'TreeRank': - tree_model = get_tree_model(tree) + tree_model = get_treedefitem_model(tree) tree_lower = tree.lower() def _build_filter_kwargs(rank_name: str, treedef_id: Optional[int] = None) -> Dict[str, Any]: @@ -77,7 +77,7 @@ def _get_treedef_id( return TreeRank(rank_name, treedef_id, tree_lower) def check_rank(self) -> bool: - tree_model = get_tree_model(self.tree) + tree_model = get_treedefitem_model(self.tree) rank = tree_model.objects.filter(name=self.rank_name, treedef_id=self.treedef_id) return rank.exists() and rank.count() == 1 @@ -91,7 +91,7 @@ def tree_rank_record(self) -> 'TreeRankRecord': assert self.tree is not None and self.tree.lower() in SPECIFY_TREES, "Tree is required" return TreeRankRecord(self.rank_name, self.treedef_id) -def get_tree_model(tree: str): +def get_treedefitem_model(tree: str): return getattr(models, tree.lower().title() + 'treedefitem') class TreeRankRecord(NamedTuple): @@ -158,10 +158,54 @@ def disambiguate(self, disambiguation: DA) -> "ScopedTreeRecord": def get_treedefs(self) -> Set: return set([self.treedef]) + + def rescope_tree_from_row(self, row: Row) -> Tuple["ScopedTreeRecord", Optional["WorkBenchParseFailure"]]: + tree_rank_model = get_treedefitem_model(self.name) + tree_node_model = getattr(models, self.name.lower().title()) + + # Create a mapping of rank names to their treedef IDs + ranks = {rank.rank_name: rank.treedef_id for rank in self.ranks.keys()} + rank_names = set(ranks.keys()) + + # Find non-null rank columns in the row + ranks_in_row_not_null = { + col + for col, value in row.items() + if value is not None and value != "" and col in rank_names + } + + # Handle cases with multiple or no ranks in the row + if len(ranks_in_row_not_null) > 1: + return self, WorkBenchParseFailure('multipleRanksInRow', {}, list(ranks_in_row_not_null)[0]) + if not ranks_in_row_not_null: + return self, WorkBenchParseFailure('noRanksInRow', {}, '') + + # Get the target rank name and its treedef ID + target_rank_name = ranks_in_row_not_null.pop() + target_rank_treedef_id = ranks[target_rank_name] + + # Query for the target rank treedef item + treedefitem_query = tree_rank_model.objects.filter(name=target_rank_name, treedef_id=target_rank_treedef_id) + + if not treedefitem_query.exists() or treedefitem_query.count() != 1: + return self, WorkBenchParseFailure('invalidRank', {'rank': target_rank_name}, target_rank_name) + + # Fetch the required data + treedefitems = list(tree_rank_model.objects.filter(treedef_id=target_rank_treedef_id)) + root = tree_node_model.objects.filter(definition_id=target_rank_treedef_id, parent=None).first() + target_rank_treedef = treedefitem_query.first().treedef + + # Return the updated ScopedTreeRecord + return self._replace(treedef=target_rank_treedef, treedefitems=treedefitems, root=root), None def bind(self, collection, row: Row, uploadingAgentId: Optional[int], auditor: Auditor, cache: Optional[Dict]=None, row_index: Optional[int] = None) -> Union["BoundTreeRecord", ParseFailures]: parsedFields: Dict[str, List[ParseResult]] = {} parseFails: List[WorkBenchParseFailure] = [] + + rescoped_tree_record, parse_fail = self.rescope_tree_from_row(row) + if parse_fail: + parseFails.append(parse_fail) + for tree_rank_record, cols in self.ranks.items(): nameColumn = cols['name'] presults, pfails = parse_many(collection, self.name, cols, row) @@ -180,9 +224,9 @@ def bind(self, collection, row: Row, uploadingAgentId: Optional[int], auditor: A return BoundTreeRecord( name=self.name, - treedef=self.treedef, - treedefitems=self.treedefitems, - root=self.root, + treedef=rescoped_tree_record.treedef, + treedefitems=rescoped_tree_record.treedefitems, + root=rescoped_tree_record.root, disambiguation=self.disambiguation, parsedFields=parsedFields, uploadingAgentId=uploadingAgentId, diff --git a/specifyweb/workbench/upload/upload_plan_schema.py b/specifyweb/workbench/upload/upload_plan_schema.py index 645e3eb71d7..a9034b1f57d 100644 --- a/specifyweb/workbench/upload/upload_plan_schema.py +++ b/specifyweb/workbench/upload/upload_plan_schema.py @@ -6,7 +6,7 @@ from .upload_table import DeferredScopeUploadTable, UploadTable, OneToOneTable, MustMatchTable from .tomany import ToManyRecord -from .treerecord import TreeRank, TreeRankRecord, TreeRecord, MustMatchTreeRecord, get_tree_model +from .treerecord import TreeRank, TreeRankRecord, TreeRecord, MustMatchTreeRecord from .uploadable import Uploadable from .column_options import ColumnOptions from .scoping import DEFERRED_SCOPING From e33c9cf6ec2db0ca180ae45a5aa9ae63acfe16c5 Mon Sep 17 00:00:00 2001 From: alec_dev Date: Tue, 3 Sep 2024 14:59:04 -0500 Subject: [PATCH 41/78] fix determinations naming bug --- specifyweb/workbench/upload/treerecord.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/specifyweb/workbench/upload/treerecord.py b/specifyweb/workbench/upload/treerecord.py index 4db1eb4f051..feea950d6c9 100644 --- a/specifyweb/workbench/upload/treerecord.py +++ b/specifyweb/workbench/upload/treerecord.py @@ -490,6 +490,8 @@ def _upload(self, to_upload: List[TreeDefItemWithParseResults], matched: Union[M def _do_insert(self, model, **kwargs): obj = model(**kwargs) + if hasattr(obj, 'fullname') and obj.fullname is None: + obj.fullname = obj.name obj.save(skip_tree_extras=True) return obj From 93810b00f3cd7c1b350e14f2075531083792c982 Mon Sep 17 00:00:00 2001 From: alec_dev Date: Tue, 3 Sep 2024 15:54:36 -0500 Subject: [PATCH 42/78] fix unit tests --- specifyweb/workbench/upload/treerecord.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/specifyweb/workbench/upload/treerecord.py b/specifyweb/workbench/upload/treerecord.py index feea950d6c9..b5ce724101e 100644 --- a/specifyweb/workbench/upload/treerecord.py +++ b/specifyweb/workbench/upload/treerecord.py @@ -163,6 +163,10 @@ def rescope_tree_from_row(self, row: Row) -> Tuple["ScopedTreeRecord", Optional[ tree_rank_model = get_treedefitem_model(self.name) tree_node_model = getattr(models, self.name.lower().title()) + # Do nothing if there is only one tree involved + if len(set([tr.treedef_id for tr in self.ranks.keys()])) == 1: + return self, None + # Create a mapping of rank names to their treedef IDs ranks = {rank.rank_name: rank.treedef_id for rank in self.ranks.keys()} rank_names = set(ranks.keys()) From 99afa31f1ec0e2aa632ece06cb26dae0873c32c0 Mon Sep 17 00:00:00 2001 From: alec_dev Date: Tue, 3 Sep 2024 17:51:59 -0500 Subject: [PATCH 43/78] better handle noRanksInRow --- specifyweb/workbench/upload/treerecord.py | 63 +++++++++++++++-------- 1 file changed, 41 insertions(+), 22 deletions(-) diff --git a/specifyweb/workbench/upload/treerecord.py b/specifyweb/workbench/upload/treerecord.py index b5ce724101e..e02027abc53 100644 --- a/specifyweb/workbench/upload/treerecord.py +++ b/specifyweb/workbench/upload/treerecord.py @@ -160,46 +160,65 @@ def get_treedefs(self) -> Set: return set([self.treedef]) def rescope_tree_from_row(self, row: Row) -> Tuple["ScopedTreeRecord", Optional["WorkBenchParseFailure"]]: - tree_rank_model = get_treedefitem_model(self.name) - tree_node_model = getattr(models, self.name.lower().title()) + def fetch_tree_node_model(name: str): + return getattr(models, name.lower().title()) - # Do nothing if there is only one tree involved - if len(set([tr.treedef_id for tr in self.ranks.keys()])) == 1: + def fetch_treedefitem_model(name: str): + return get_treedefitem_model(name) + + def get_ranks_mapping(ranks): + return {rank.rank_name: rank.treedef_id for rank in ranks.keys()} + + def find_non_null_ranks_in_row(row, rank_names): + return { + col + for col, value in row.items() + if value is not None and value != "" and col in rank_names + } + + def handle_no_ranks_in_row(): + if self.treedef is not None: + return self, None + else: + first_rank = next(iter(self.ranks.keys())) + target_rank_treedef_id = first_rank.treedef_id + target_rank_treedef = tree_node_model.objects.filter(definition_id=target_rank_treedef_id, parent=None).first() + treedefitems = list(tree_rank_model.objects.filter(treedef_id=target_rank_treedef_id)) + root = tree_node_model.objects.filter(definition_id=target_rank_treedef_id, parent=None).first() + return self._replace(treedef=target_rank_treedef, treedefitems=treedefitems, root=root), None + + def handle_multiple_ranks_in_row(ranks_in_row_not_null): + return self, WorkBenchParseFailure('multipleRanksInRow', {}, list(ranks_in_row_not_null)[0]) + + def handle_invalid_rank(target_rank_name): + return self, WorkBenchParseFailure('invalidRank', {'rank': target_rank_name}, target_rank_name) + + tree_rank_model = fetch_treedefitem_model(self.name) + tree_node_model = fetch_tree_node_model(self.name) + + if len(set(tr.treedef_id for tr in self.ranks.keys())) == 1: return self, None - # Create a mapping of rank names to their treedef IDs - ranks = {rank.rank_name: rank.treedef_id for rank in self.ranks.keys()} + ranks = get_ranks_mapping(self.ranks) rank_names = set(ranks.keys()) + ranks_in_row_not_null = find_non_null_ranks_in_row(row, rank_names) - # Find non-null rank columns in the row - ranks_in_row_not_null = { - col - for col, value in row.items() - if value is not None and value != "" and col in rank_names - } - - # Handle cases with multiple or no ranks in the row if len(ranks_in_row_not_null) > 1: - return self, WorkBenchParseFailure('multipleRanksInRow', {}, list(ranks_in_row_not_null)[0]) + return handle_multiple_ranks_in_row(ranks_in_row_not_null) if not ranks_in_row_not_null: - return self, WorkBenchParseFailure('noRanksInRow', {}, '') + return handle_no_ranks_in_row() - # Get the target rank name and its treedef ID target_rank_name = ranks_in_row_not_null.pop() target_rank_treedef_id = ranks[target_rank_name] - - # Query for the target rank treedef item treedefitem_query = tree_rank_model.objects.filter(name=target_rank_name, treedef_id=target_rank_treedef_id) if not treedefitem_query.exists() or treedefitem_query.count() != 1: - return self, WorkBenchParseFailure('invalidRank', {'rank': target_rank_name}, target_rank_name) + return handle_invalid_rank(target_rank_name) - # Fetch the required data treedefitems = list(tree_rank_model.objects.filter(treedef_id=target_rank_treedef_id)) root = tree_node_model.objects.filter(definition_id=target_rank_treedef_id, parent=None).first() target_rank_treedef = treedefitem_query.first().treedef - # Return the updated ScopedTreeRecord return self._replace(treedef=target_rank_treedef, treedefitems=treedefitems, root=root), None def bind(self, collection, row: Row, uploadingAgentId: Optional[int], auditor: Auditor, cache: Optional[Dict]=None, row_index: Optional[int] = None) -> Union["BoundTreeRecord", ParseFailures]: From c4ac5a8805fce5bc7831f6af9055eebf8d5ce7ff Mon Sep 17 00:00:00 2001 From: alec_dev Date: Tue, 3 Sep 2024 18:10:30 -0500 Subject: [PATCH 44/78] order by rankid --- specifyweb/workbench/upload/treerecord.py | 29 ++++++++++++++++++----- 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/specifyweb/workbench/upload/treerecord.py b/specifyweb/workbench/upload/treerecord.py index e02027abc53..869850cae2f 100644 --- a/specifyweb/workbench/upload/treerecord.py +++ b/specifyweb/workbench/upload/treerecord.py @@ -158,7 +158,7 @@ def disambiguate(self, disambiguation: DA) -> "ScopedTreeRecord": def get_treedefs(self) -> Set: return set([self.treedef]) - + def rescope_tree_from_row(self, row: Row) -> Tuple["ScopedTreeRecord", Optional["WorkBenchParseFailure"]]: def fetch_tree_node_model(name: str): return getattr(models, name.lower().title()) @@ -182,8 +182,14 @@ def handle_no_ranks_in_row(): else: first_rank = next(iter(self.ranks.keys())) target_rank_treedef_id = first_rank.treedef_id - target_rank_treedef = tree_node_model.objects.filter(definition_id=target_rank_treedef_id, parent=None).first() - treedefitems = list(tree_rank_model.objects.filter(treedef_id=target_rank_treedef_id)) + target_rank_treedef = tree_node_model.objects.filter( + definition_id=target_rank_treedef_id, parent=None + ).first() + treedefitems = list( + tree_rank_model.objects.filter( + treedef_id=target_rank_treedef_id + ).order_by("rankid") + ) root = tree_node_model.objects.filter(definition_id=target_rank_treedef_id, parent=None).first() return self._replace(treedef=target_rank_treedef, treedefitems=treedefitems, root=root), None @@ -210,18 +216,28 @@ def handle_invalid_rank(target_rank_name): target_rank_name = ranks_in_row_not_null.pop() target_rank_treedef_id = ranks[target_rank_name] - treedefitem_query = tree_rank_model.objects.filter(name=target_rank_name, treedef_id=target_rank_treedef_id) + treedefitem_query = tree_rank_model.objects.filter( + name=target_rank_name, treedef_id=target_rank_treedef_id + ).order_by("rankid") if not treedefitem_query.exists() or treedefitem_query.count() != 1: return handle_invalid_rank(target_rank_name) - treedefitems = list(tree_rank_model.objects.filter(treedef_id=target_rank_treedef_id)) + treedefitems = list(tree_rank_model.objects.filter(treedef_id=target_rank_treedef_id).order_by("rankid")) root = tree_node_model.objects.filter(definition_id=target_rank_treedef_id, parent=None).first() target_rank_treedef = treedefitem_query.first().treedef return self._replace(treedef=target_rank_treedef, treedefitems=treedefitems, root=root), None - def bind(self, collection, row: Row, uploadingAgentId: Optional[int], auditor: Auditor, cache: Optional[Dict]=None, row_index: Optional[int] = None) -> Union["BoundTreeRecord", ParseFailures]: + def bind( + self, + collection, + row: Row, + uploadingAgentId: Optional[int], + auditor: Auditor, + cache: Optional[Dict] = None, + row_index: Optional[int] = None, + ) -> Union["BoundTreeRecord", ParseFailures]: parsedFields: Dict[str, List[ParseResult]] = {} parseFails: List[WorkBenchParseFailure] = [] @@ -257,6 +273,7 @@ def bind(self, collection, row: Row, uploadingAgentId: Optional[int], auditor: A cache=cache, ) + class MustMatchTreeRecord(TreeRecord): def apply_scoping(self, collection) -> "ScopedMustMatchTreeRecord": s = super().apply_scoping(collection) From 61034914593ab05a8dfe41a6314bd9ec51e7a18a Mon Sep 17 00:00:00 2001 From: alec_dev Date: Tue, 3 Sep 2024 18:14:28 -0500 Subject: [PATCH 45/78] remove unneeded conditional in handle_no_ranks_in_row --- specifyweb/workbench/upload/treerecord.py | 25 +++++++++++------------ 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/specifyweb/workbench/upload/treerecord.py b/specifyweb/workbench/upload/treerecord.py index 869850cae2f..d159c5e2bb1 100644 --- a/specifyweb/workbench/upload/treerecord.py +++ b/specifyweb/workbench/upload/treerecord.py @@ -179,19 +179,18 @@ def find_non_null_ranks_in_row(row, rank_names): def handle_no_ranks_in_row(): if self.treedef is not None: return self, None - else: - first_rank = next(iter(self.ranks.keys())) - target_rank_treedef_id = first_rank.treedef_id - target_rank_treedef = tree_node_model.objects.filter( - definition_id=target_rank_treedef_id, parent=None - ).first() - treedefitems = list( - tree_rank_model.objects.filter( - treedef_id=target_rank_treedef_id - ).order_by("rankid") - ) - root = tree_node_model.objects.filter(definition_id=target_rank_treedef_id, parent=None).first() - return self._replace(treedef=target_rank_treedef, treedefitems=treedefitems, root=root), None + first_rank = next(iter(self.ranks.keys())) + target_rank_treedef_id = first_rank.treedef_id + target_rank_treedef = tree_node_model.objects.filter( + definition_id=target_rank_treedef_id, parent=None + ).first() + treedefitems = list( + tree_rank_model.objects.filter( + treedef_id=target_rank_treedef_id + ).order_by("rankid") + ) + root = tree_node_model.objects.filter(definition_id=target_rank_treedef_id, parent=None).first() + return self._replace(treedef=target_rank_treedef, treedefitems=treedefitems, root=root), None def handle_multiple_ranks_in_row(ranks_in_row_not_null): return self, WorkBenchParseFailure('multipleRanksInRow', {}, list(ranks_in_row_not_null)[0]) From 508cbef3599908771996fb481563e908ce49cb5d Mon Sep 17 00:00:00 2001 From: alec_dev Date: Wed, 4 Sep 2024 01:41:51 -0500 Subject: [PATCH 46/78] new rank key request formatting --- specifyweb/workbench/upload/treerecord.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/specifyweb/workbench/upload/treerecord.py b/specifyweb/workbench/upload/treerecord.py index d159c5e2bb1..d488d709f2b 100644 --- a/specifyweb/workbench/upload/treerecord.py +++ b/specifyweb/workbench/upload/treerecord.py @@ -3,6 +3,7 @@ """ import logging +import re from typing import List, Dict, Any, Tuple, NamedTuple, Optional, Union, Set from django.db import transaction, IntegrityError @@ -73,7 +74,12 @@ def _get_treedef_id( return first_item.treedef_id - treedef_id = _get_treedef_id(rank_name, tree, treedef_id, base_treedef_id) + if re.match(r'.*~>\d+$', rank_name): + rank_name, treedef_id = re.split(r'~>', rank_name) + treedef_id = int(treedef_id) + else: + treedef_id = _get_treedef_id(rank_name, tree, treedef_id, base_treedef_id) + return TreeRank(rank_name, treedef_id, tree_lower) def check_rank(self) -> bool: From 8942c8776f0013da77d327dfdebe34e88aea9ce0 Mon Sep 17 00:00:00 2001 From: alec_dev Date: Wed, 4 Sep 2024 09:52:30 -0500 Subject: [PATCH 47/78] fix type error --- specifyweb/workbench/upload/treerecord.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specifyweb/workbench/upload/treerecord.py b/specifyweb/workbench/upload/treerecord.py index d488d709f2b..dc36841daeb 100644 --- a/specifyweb/workbench/upload/treerecord.py +++ b/specifyweb/workbench/upload/treerecord.py @@ -75,8 +75,8 @@ def _get_treedef_id( return first_item.treedef_id if re.match(r'.*~>\d+$', rank_name): - rank_name, treedef_id = re.split(r'~>', rank_name) - treedef_id = int(treedef_id) + rank_name, treedef_id_str = re.split(r'~>', rank_name) + treedef_id = int(treedef_id_str) else: treedef_id = _get_treedef_id(rank_name, tree, treedef_id, base_treedef_id) From 79bbbccc08c64623a5f9b48fdf85671ddc782625 Mon Sep 17 00:00:00 2001 From: Sharad S Date: Wed, 4 Sep 2024 12:55:54 -0400 Subject: [PATCH 48/78] Modify upload plan request to avoid duplicate rank keys --- .../components/WbPlanView/mappingHelpers.ts | 22 +++++++++++++++++++ .../WbPlanView/uploadPlanBuilder.ts | 6 +++-- .../components/WbPlanView/uploadPlanParser.ts | 4 ++-- 3 files changed, 28 insertions(+), 4 deletions(-) diff --git a/specifyweb/frontend/js_src/lib/components/WbPlanView/mappingHelpers.ts b/specifyweb/frontend/js_src/lib/components/WbPlanView/mappingHelpers.ts index dc65961182d..401a9623288 100644 --- a/specifyweb/frontend/js_src/lib/components/WbPlanView/mappingHelpers.ts +++ b/specifyweb/frontend/js_src/lib/components/WbPlanView/mappingHelpers.ts @@ -111,6 +111,28 @@ export const formatTreeDefinition = (definitionName: string): string => export const formatTreeRank = (rankName: string): string => `${schema.treeRankSymbol}${rankName}`; +// Delimiter used for rank name keys i.e: ~> +const RANK_KEY_DELIMITER = '~>'; + +/** + * Returns a formatted tree rank name along with its tree id: (e.x $Kingdom => $Kingdom~>1) + * Used for generating unique key names when there are multiple trees with the same rank name. + * Opposite of getRankNameWithoutTreeId + * See: https://github.com/specify/specify7/pull/5091#issuecomment-2328037741 + */ +export const formatTreeRankWithTreeId = ( + rankName: string, + treeId: number +): string => `${rankName}${RANK_KEY_DELIMITER}${treeId}`; + +/** + * Returns the tree rank name after stripping its tree id: (e.x Kingdom~>1 => Kingdom) + * NOTE: Does not consider whether rankName is formatted with $ or not. + * Opposite of formatTreeRankWithTreeId + */ +export const getRankNameWithoutTreeId = (rankName: string): string => + rankName.split(RANK_KEY_DELIMITER)[0]; + // Match fields names like startDate_fullDate, but not _formatted export const valueIsPartialField = (value: string): boolean => value.includes(schema.fieldPartSeparator) && diff --git a/specifyweb/frontend/js_src/lib/components/WbPlanView/uploadPlanBuilder.ts b/specifyweb/frontend/js_src/lib/components/WbPlanView/uploadPlanBuilder.ts index b839b5719b1..493933674dd 100644 --- a/specifyweb/frontend/js_src/lib/components/WbPlanView/uploadPlanBuilder.ts +++ b/specifyweb/frontend/js_src/lib/components/WbPlanView/uploadPlanBuilder.ts @@ -5,7 +5,7 @@ import { strictGetTable } from '../DataModel/tables'; import type { Tables } from '../DataModel/types'; import { getTreeDefinitions, isTreeTable } from '../InitialContext/treeRanks'; import { defaultColumnOptions } from './linesGetter'; -import type { SplitMappingPath } from './mappingHelpers'; +import { formatTreeRankWithTreeId, SplitMappingPath } from './mappingHelpers'; import { getNameFromTreeDefinitionName, valueIsTreeDefinition, @@ -71,7 +71,9 @@ const toTreeRecordVariety = (lines: RA): TreeRecord => { )?.definition.id; return [ - rankName, + treeId === undefined + ? rankName + : formatTreeRankWithTreeId(rankName, treeId), { treeNodeCols: toTreeRecordRanks(rankMappedFields), treeId, diff --git a/specifyweb/frontend/js_src/lib/components/WbPlanView/uploadPlanParser.ts b/specifyweb/frontend/js_src/lib/components/WbPlanView/uploadPlanParser.ts index b1378674013..e3e54e73127 100644 --- a/specifyweb/frontend/js_src/lib/components/WbPlanView/uploadPlanParser.ts +++ b/specifyweb/frontend/js_src/lib/components/WbPlanView/uploadPlanParser.ts @@ -6,7 +6,7 @@ import { softFail } from '../Errors/Crash'; import { getTreeDefinitions } from '../InitialContext/treeRanks'; import { defaultColumnOptions } from './linesGetter'; import type { MappingPath } from './Mapper'; -import type { SplitMappingPath } from './mappingHelpers'; +import { getRankNameWithoutTreeId, SplitMappingPath } from './mappingHelpers'; import { formatTreeDefinition } from './mappingHelpers'; import { formatToManyIndex, formatTreeRank } from './mappingHelpers'; @@ -82,7 +82,7 @@ const parseTree = ( getTreeDefinitions('Taxon', 'all').length > 1 ? [resolveTreeId(rankData.treeId)] : []), - formatTreeRank(rankName), + formatTreeRank(getRankNameWithoutTreeId(rankName)), ] ) ); From eb503a4a53bd8b7cb2c4c31dba0f96fa01cd2e2b Mon Sep 17 00:00:00 2001 From: Sharad S Date: Wed, 4 Sep 2024 15:21:42 -0400 Subject: [PATCH 49/78] fix upload plan test --- .../js_src/lib/tests/fixtures/uploadplan.1.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/specifyweb/frontend/js_src/lib/tests/fixtures/uploadplan.1.json b/specifyweb/frontend/js_src/lib/tests/fixtures/uploadplan.1.json index a639bc1327d..26476e857f6 100644 --- a/specifyweb/frontend/js_src/lib/tests/fixtures/uploadplan.1.json +++ b/specifyweb/frontend/js_src/lib/tests/fixtures/uploadplan.1.json @@ -275,7 +275,7 @@ "taxon": { "mustMatchTreeRecord": { "ranks": { - "Class": { + "Class~>1": { "treeNodeCols": { "name": { "matchBehavior": "ignoreAlways", @@ -286,32 +286,32 @@ }, "treeId": 1 }, - "Family": { + "Family~>1": { "treeNodeCols": { "name": "Family" }, "treeId": 1 }, - "Genus": { + "Genus~>1": { "treeNodeCols": { "name": "Genus" }, "treeId": 1 }, - "Subgenus": { + "Subgenus~>1": { "treeNodeCols": { "name": "Subgenus" }, "treeId": 1 }, - "Species": { + "Species~>1": { "treeNodeCols": { "name": "Species", "author": "Species Author" }, "treeId": 1 }, - "Subspecies": { + "Subspecies~>1": { "treeNodeCols": { "name": "Subspecies", "author": "Subspecies Author" From 655a793e79081f4b6028ce54aa3a8e6eea4b941a Mon Sep 17 00:00:00 2001 From: alec_dev Date: Thu, 5 Sep 2024 11:56:16 -0500 Subject: [PATCH 50/78] reformulate rescope_tree_from_row --- specifyweb/workbench/upload/treerecord.py | 107 +++++++++++++++++++++- 1 file changed, 104 insertions(+), 3 deletions(-) diff --git a/specifyweb/workbench/upload/treerecord.py b/specifyweb/workbench/upload/treerecord.py index dc36841daeb..77c6cf9565e 100644 --- a/specifyweb/workbench/upload/treerecord.py +++ b/specifyweb/workbench/upload/treerecord.py @@ -2,6 +2,7 @@ For uploading tree records. """ +from collections import namedtuple import logging import re from typing import List, Dict, Any, Tuple, NamedTuple, Optional, Union, Set @@ -76,7 +77,7 @@ def _get_treedef_id( if re.match(r'.*~>\d+$', rank_name): rank_name, treedef_id_str = re.split(r'~>', rank_name) - treedef_id = int(treedef_id_str) + treedef_id = _get_treedef_id(rank_name, tree, int(treedef_id_str), base_treedef_id) else: treedef_id = _get_treedef_id(rank_name, tree, treedef_id, base_treedef_id) @@ -100,6 +101,9 @@ def tree_rank_record(self) -> 'TreeRankRecord': def get_treedefitem_model(tree: str): return getattr(models, tree.lower().title() + 'treedefitem') +def get_treedef_model(tree: str): + return getattr(models, tree.lower().title() + 'treedef') + class TreeRankRecord(NamedTuple): rank_name: str treedef_id: int @@ -163,9 +167,16 @@ def disambiguate(self, disambiguation: DA) -> "ScopedTreeRecord": return self._replace(disambiguation=disambiguation.disambiguate_tree()) if disambiguation is not None else self def get_treedefs(self) -> Set: - return set([self.treedef]) + # return set([self.treedef]) # old way - def rescope_tree_from_row(self, row: Row) -> Tuple["ScopedTreeRecord", Optional["WorkBenchParseFailure"]]: + tree_def_model = get_treedef_model(self.name) + treedefids = set([tree_rank_record.treedef_id for tree_rank_record in self.ranks.keys()]) + # return set([tree_rank_record.treedef_id for tree_rank_record in self.ranks.keys()]) + # return tree_rank_model.objects.filter(treedef_id__in=treedefids) + return set(tree_def_model.objects.filter(id__in=treedefids)) + + # Remove this old version of the funciton + def rescope_tree_from_row_old(self, row: Row) -> Tuple["ScopedTreeRecord", Optional["WorkBenchParseFailure"]]: def fetch_tree_node_model(name: str): return getattr(models, name.lower().title()) @@ -204,12 +215,30 @@ def handle_multiple_ranks_in_row(ranks_in_row_not_null): def handle_invalid_rank(target_rank_name): return self, WorkBenchParseFailure('invalidRank', {'rank': target_rank_name}, target_rank_name) + # def handle_extended_rank_name(rank_col_name): + # extracted_rank_name = None + # if ' - ' in rank_col_name: + # extracted_rank_name, treedef_name = rank_col_name.split(' - ') + # elif '~>' in rank_col_name: + # extracted_rank_name, treedef_name = rank_col_name.split('~>') + # if extracted_rank_name in set([rank.rank_name for rank in self.ranks]): + # return extracted_rank_name + # return rank_col_name + + # def modify_row_for_extended_rank_names(row): + # row_modified = {} + # for col_name in row.keys(): + # col_name = handle_extended_rank_name(col_name) + # row_modified[col_name] = row[col_name] + # return row_modified + tree_rank_model = fetch_treedefitem_model(self.name) tree_node_model = fetch_tree_node_model(self.name) if len(set(tr.treedef_id for tr in self.ranks.keys())) == 1: return self, None + # mod_row = modify_row_for_extended_rank_names(row) ranks = get_ranks_mapping(self.ranks) rank_names = set(ranks.keys()) ranks_in_row_not_null = find_non_null_ranks_in_row(row, rank_names) @@ -234,6 +263,78 @@ def handle_invalid_rank(target_rank_name): return self._replace(treedef=target_rank_treedef, treedefitems=treedefitems, root=root), None + RankColumn = namedtuple('RankColumn', ['treedef_id', 'treedefitem_name', 'tree_node_attribute', 'upload_value']) + + def _get_not_null_ranks_columns_in_row(self, row: Row) -> List["RankColumn"]: + # Get rank columns that are not null in the row + RankColumn = self.__class__.RankColumn + ranks_columns_in_row_not_null: List["RankColumn"] = [] + for row_key, row_value in row.items(): + if not row_value: + continue + for tree_rank_record, cols in self.ranks.items(): + for col_name, col_opts in cols.items(): + formatted_column = f'{col_opts.column} - {col_name}' if col_name != 'name' else col_opts.column + if formatted_column == row_key: + ranks_columns_in_row_not_null.append( + RankColumn( + tree_rank_record.treedef_id, # treedefid + tree_rank_record.rank_name, # treedefitem_name + col_name, # tree_node_attribute + row_value, # upload_value + ) + ) + break + return ranks_columns_in_row_not_null + + def _filter_target_rank_columns(self, ranks_columns_in_row_not_null, target_rank_treedef_id) -> List["RankColumn"]: + # Filter ranks_columns_in_row_not_null to only include columns that are part of the target treedef + return list( + filter( + lambda rank_column: rank_column.treedef_id == target_rank_treedef_id, + ranks_columns_in_row_not_null, + ) + ) + + def rescope_tree_from_row(self, row: Row) -> Tuple["ScopedTreeRecord", Optional["WorkBenchParseFailure"]]: + # Starting out with a row with tree data. + # The row can have multiple tree ranks, and each rank can be associated with multiple columns, like name, authoer, etc. + # First need to match each rank column to a treedefitem. + # Second need to narrow down to the target rank based on which values in the row are not null. + # Lastly, need to return a new ScopedTreeRecord with the correct treedef, treedefitems, and root for this row. + + tree_def_model = get_treedef_model(self.name) + tree_rank_model = get_treedefitem_model(self.name) + tree_node_model = getattr(models, self.name.lower().title()) + + if len(set(tr.treedef_id for tr in self.ranks.keys())) == 1: + return self, None + + # Get rank columns that are not null in the row + ranks_columns_in_row_not_null = self._get_not_null_ranks_columns_in_row(row) + + # Determine the target treedef based on the columns that are not null + targeted_treedefids = set([rank_column.treedef_id for rank_column in ranks_columns_in_row_not_null]) + if targeted_treedefids is None or len(targeted_treedefids) == 0: + # return self, WorkBenchParseFailure('noRanksInRow', {}, None) + return self, None + elif len(targeted_treedefids) > 1: + # return self, WorkBenchParseFailure('multipleRanksInRow', {}, list(ranks_in_row_not_null)[0]) + logger.warning(f"Multiple treedefs found in row: {targeted_treedefids}") + + target_rank_treedef_id = targeted_treedefids.pop() + target_rank_treedef = tree_def_model.objects.get(id=target_rank_treedef_id) + + # Filter ranks_columns_in_row_not_null to only include columns that are part of the target treedef + # ranks_columns = self._filter_target_rank_columns(ranks_columns_in_row_not_null, target_rank_treedef_id) + + # Based on the target treedef, get the treedefitems and root for the tree + treedefitems = list(tree_rank_model.objects.filter(treedef_id=target_rank_treedef_id).order_by("rankid")) + root = tree_node_model.objects.filter(definition_id=target_rank_treedef_id, parent=None).first() + + # Return a new ScopedTreeRecord with the correct treedefitem and treedef + return self._replace(treedef=target_rank_treedef, treedefitems=treedefitems, root=root), None + def bind( self, collection, From 9babfb3148b07375f20719476bd61ec01167903d Mon Sep 17 00:00:00 2001 From: Sharad S Date: Thu, 5 Sep 2024 13:55:18 -0400 Subject: [PATCH 51/78] Change tree headers when rank name is ambiguous --- .../js_src/lib/components/WbPlanView/headerHelper.ts | 12 +++++++++++- .../lib/components/WbPlanView/mappingPreview.ts | 7 ++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/specifyweb/frontend/js_src/lib/components/WbPlanView/headerHelper.ts b/specifyweb/frontend/js_src/lib/components/WbPlanView/headerHelper.ts index 97915632fd8..ab23c77f7a4 100644 --- a/specifyweb/frontend/js_src/lib/components/WbPlanView/headerHelper.ts +++ b/specifyweb/frontend/js_src/lib/components/WbPlanView/headerHelper.ts @@ -9,6 +9,7 @@ import type { RA } from '../../utils/types'; import { getUniqueName } from '../../utils/uniquifyName'; import type { Tables } from '../DataModel/types'; import type { MappingLine } from './Mapper'; +import { valueIsTreeRank } from './mappingHelpers'; import { generateMappingPathPreview } from './mappingPreview'; export function uniquifyHeaders( @@ -34,13 +35,22 @@ export function renameNewlyCreatedHeaders( headers: RA, lines: RA ): RA { + const duplicateRanks = new Set( + lines + .flatMap(({ mappingPath }) => mappingPath) + .filter( + (rank, index, ranks) => + valueIsTreeRank(rank) && ranks.indexOf(rank) !== index + ) + ); + const generatedHeaderPreviews = Object.fromEntries( lines .map((line, index) => ({ line, index })) .filter(({ line }) => !headers.includes(line.headerName)) .map(({ line, index }) => [ index, - generateMappingPathPreview(baseTableName, line.mappingPath), + generateMappingPathPreview(baseTableName, line.mappingPath, duplicateRanks), ]) ); diff --git a/specifyweb/frontend/js_src/lib/components/WbPlanView/mappingPreview.ts b/specifyweb/frontend/js_src/lib/components/WbPlanView/mappingPreview.ts index 3a27a0fade9..e4e6924327f 100644 --- a/specifyweb/frontend/js_src/lib/components/WbPlanView/mappingPreview.ts +++ b/specifyweb/frontend/js_src/lib/components/WbPlanView/mappingPreview.ts @@ -83,7 +83,8 @@ const mappingPathSubset = ( */ export function generateMappingPathPreview( baseTableName: keyof Tables, - mappingPath: MappingPath + mappingPath: MappingPath, + duplicateRanks: Set ): string { if (mappingPath.length === 0) return strictGetTable(baseTableName).label; @@ -173,6 +174,10 @@ export function generateMappingPathPreview( ? [isAnyRank ? parentTableName : tableOrRankName] : tableNameFormatted), fieldNameFormatted, + ...(valueIsTreeRank(databaseTableOrRankName) && + duplicateRanks.has(databaseTableOrRankName) + ? [parentTableName] + : []), toManyIndexFormatted, ]) .filter(Boolean) From ca338dcce2860284cef8a86e39171458e358dc02 Mon Sep 17 00:00:00 2001 From: Sharad S Date: Thu, 5 Sep 2024 13:59:41 -0400 Subject: [PATCH 52/78] fix typecheck --- .../js_src/lib/components/WbPlanView/mappingPreview.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specifyweb/frontend/js_src/lib/components/WbPlanView/mappingPreview.ts b/specifyweb/frontend/js_src/lib/components/WbPlanView/mappingPreview.ts index e4e6924327f..6d2405cbc94 100644 --- a/specifyweb/frontend/js_src/lib/components/WbPlanView/mappingPreview.ts +++ b/specifyweb/frontend/js_src/lib/components/WbPlanView/mappingPreview.ts @@ -84,7 +84,7 @@ const mappingPathSubset = ( export function generateMappingPathPreview( baseTableName: keyof Tables, mappingPath: MappingPath, - duplicateRanks: Set + duplicateRanks?: Set ): string { if (mappingPath.length === 0) return strictGetTable(baseTableName).label; @@ -175,7 +175,7 @@ export function generateMappingPathPreview( : tableNameFormatted), fieldNameFormatted, ...(valueIsTreeRank(databaseTableOrRankName) && - duplicateRanks.has(databaseTableOrRankName) + duplicateRanks?.has(databaseTableOrRankName) ? [parentTableName] : []), toManyIndexFormatted, From 39b45ce69e905754a9be7eba057c7fbb35c3d2d4 Mon Sep 17 00:00:00 2001 From: Sharad S Date: Thu, 5 Sep 2024 18:03:14 +0000 Subject: [PATCH 53/78] Lint code with ESLint and Prettier Triggered by ca338dcce2860284cef8a86e39171458e358dc02 on branch refs/heads/issue-4980 --- .../frontend/js_src/lib/components/WbPlanView/mappingPreview.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specifyweb/frontend/js_src/lib/components/WbPlanView/mappingPreview.ts b/specifyweb/frontend/js_src/lib/components/WbPlanView/mappingPreview.ts index 6d2405cbc94..083b38ddc8c 100644 --- a/specifyweb/frontend/js_src/lib/components/WbPlanView/mappingPreview.ts +++ b/specifyweb/frontend/js_src/lib/components/WbPlanView/mappingPreview.ts @@ -84,7 +84,7 @@ const mappingPathSubset = ( export function generateMappingPathPreview( baseTableName: keyof Tables, mappingPath: MappingPath, - duplicateRanks?: Set + duplicateRanks?: ReadonlySet ): string { if (mappingPath.length === 0) return strictGetTable(baseTableName).label; From e215a77b47e1af743176ef666cb65e0ff4770a21 Mon Sep 17 00:00:00 2001 From: alec_dev Date: Thu, 5 Sep 2024 13:55:35 -0500 Subject: [PATCH 54/78] fix RankColumn typecheck --- specifyweb/workbench/upload/treerecord.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/specifyweb/workbench/upload/treerecord.py b/specifyweb/workbench/upload/treerecord.py index 77c6cf9565e..d320d9cee44 100644 --- a/specifyweb/workbench/upload/treerecord.py +++ b/specifyweb/workbench/upload/treerecord.py @@ -22,6 +22,11 @@ logger = logging.getLogger(__name__) +class RankColumn(NamedTuple): + treedef_id: int + treedefitem_name: str + tree_node_attribute: str + upload_value: str class TreeRank(NamedTuple): rank_name: str treedef_id: int @@ -261,14 +266,11 @@ def handle_invalid_rank(target_rank_name): root = tree_node_model.objects.filter(definition_id=target_rank_treedef_id, parent=None).first() target_rank_treedef = treedefitem_query.first().treedef - return self._replace(treedef=target_rank_treedef, treedefitems=treedefitems, root=root), None - - RankColumn = namedtuple('RankColumn', ['treedef_id', 'treedefitem_name', 'tree_node_attribute', 'upload_value']) + return self._replace(treedef=target_rank_treedef, treedefitems=treedefitems, root=root), None - def _get_not_null_ranks_columns_in_row(self, row: Row) -> List["RankColumn"]: + def _get_not_null_ranks_columns_in_row(self, row: Row) -> List[RankColumn]: # Get rank columns that are not null in the row - RankColumn = self.__class__.RankColumn - ranks_columns_in_row_not_null: List["RankColumn"] = [] + ranks_columns_in_row_not_null = [] for row_key, row_value in row.items(): if not row_value: continue @@ -287,7 +289,7 @@ def _get_not_null_ranks_columns_in_row(self, row: Row) -> List["RankColumn"]: break return ranks_columns_in_row_not_null - def _filter_target_rank_columns(self, ranks_columns_in_row_not_null, target_rank_treedef_id) -> List["RankColumn"]: + def _filter_target_rank_columns(self, ranks_columns_in_row_not_null, target_rank_treedef_id) -> List[RankColumn]: # Filter ranks_columns_in_row_not_null to only include columns that are part of the target treedef return list( filter( @@ -311,7 +313,7 @@ def rescope_tree_from_row(self, row: Row) -> Tuple["ScopedTreeRecord", Optional[ return self, None # Get rank columns that are not null in the row - ranks_columns_in_row_not_null = self._get_not_null_ranks_columns_in_row(row) + ranks_columns_in_row_not_null: List[RankColumn] = self._get_not_null_ranks_columns_in_row(row) # Determine the target treedef based on the columns that are not null targeted_treedefids = set([rank_column.treedef_id for rank_column in ranks_columns_in_row_not_null]) From 3595c2be9946fdf72663180a7762c658a8969aa0 Mon Sep 17 00:00:00 2001 From: alec_dev Date: Thu, 5 Sep 2024 13:58:50 -0500 Subject: [PATCH 55/78] TreeRankCell --- specifyweb/workbench/upload/treerecord.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/specifyweb/workbench/upload/treerecord.py b/specifyweb/workbench/upload/treerecord.py index d320d9cee44..9604f0a0bc6 100644 --- a/specifyweb/workbench/upload/treerecord.py +++ b/specifyweb/workbench/upload/treerecord.py @@ -22,11 +22,12 @@ logger = logging.getLogger(__name__) -class RankColumn(NamedTuple): +class TreeRankCell(NamedTuple): treedef_id: int treedefitem_name: str tree_node_attribute: str upload_value: str + class TreeRank(NamedTuple): rank_name: str treedef_id: int @@ -268,7 +269,7 @@ def handle_invalid_rank(target_rank_name): return self._replace(treedef=target_rank_treedef, treedefitems=treedefitems, root=root), None - def _get_not_null_ranks_columns_in_row(self, row: Row) -> List[RankColumn]: + def _get_not_null_ranks_columns_in_row(self, row: Row) -> List[TreeRankCell]: # Get rank columns that are not null in the row ranks_columns_in_row_not_null = [] for row_key, row_value in row.items(): @@ -279,7 +280,7 @@ def _get_not_null_ranks_columns_in_row(self, row: Row) -> List[RankColumn]: formatted_column = f'{col_opts.column} - {col_name}' if col_name != 'name' else col_opts.column if formatted_column == row_key: ranks_columns_in_row_not_null.append( - RankColumn( + TreeRankCell( tree_rank_record.treedef_id, # treedefid tree_rank_record.rank_name, # treedefitem_name col_name, # tree_node_attribute @@ -289,7 +290,7 @@ def _get_not_null_ranks_columns_in_row(self, row: Row) -> List[RankColumn]: break return ranks_columns_in_row_not_null - def _filter_target_rank_columns(self, ranks_columns_in_row_not_null, target_rank_treedef_id) -> List[RankColumn]: + def _filter_target_rank_columns(self, ranks_columns_in_row_not_null, target_rank_treedef_id) -> List[TreeRankCell]: # Filter ranks_columns_in_row_not_null to only include columns that are part of the target treedef return list( filter( @@ -313,7 +314,7 @@ def rescope_tree_from_row(self, row: Row) -> Tuple["ScopedTreeRecord", Optional[ return self, None # Get rank columns that are not null in the row - ranks_columns_in_row_not_null: List[RankColumn] = self._get_not_null_ranks_columns_in_row(row) + ranks_columns_in_row_not_null: List[TreeRankCell] = self._get_not_null_ranks_columns_in_row(row) # Determine the target treedef based on the columns that are not null targeted_treedefids = set([rank_column.treedef_id for rank_column in ranks_columns_in_row_not_null]) From 270d06f26c909446f922bc6fb6bd47c25a3a9e9a Mon Sep 17 00:00:00 2001 From: Sharad S Date: Thu, 5 Sep 2024 15:32:13 -0400 Subject: [PATCH 56/78] Fix mapping header being renamed in collections with only 1 tree --- .../components/WbPlanView/mappingPreview.ts | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/specifyweb/frontend/js_src/lib/components/WbPlanView/mappingPreview.ts b/specifyweb/frontend/js_src/lib/components/WbPlanView/mappingPreview.ts index 6d2405cbc94..2c659aaa3ac 100644 --- a/specifyweb/frontend/js_src/lib/components/WbPlanView/mappingPreview.ts +++ b/specifyweb/frontend/js_src/lib/components/WbPlanView/mappingPreview.ts @@ -22,6 +22,7 @@ import { parsePartialField, valueIsPartialField, valueIsToManyIndex, + valueIsTreeDefinition, valueIsTreeRank, } from './mappingHelpers'; import { getMappingLineData } from './navigator'; @@ -121,8 +122,11 @@ export function generateMappingPathPreview( : 1; const toManyIndexFormatted = toManyIndexNumber > 1 ? toManyIndex : undefined; - const [databaseFieldName, databaseTableOrRankName, databaseParentTableName] = - mappingPathSubset([baseTableName, ...mappingPath]); + const [ + databaseFieldName, + databaseTableOrRankName, + databaseParentTableOrTreeName, + ] = mappingPathSubset([baseTableName, ...mappingPath]); // Attributes parts of filedLables to each variable or creates one if empty const [ @@ -130,7 +134,7 @@ export function generateMappingPathPreview( tableOrRankName = camelToHuman( getNameFromTreeRankName(databaseTableOrRankName) ), - parentTableName = camelToHuman(databaseParentTableName), + parentTableOrTreeName = camelToHuman(databaseParentTableOrTreeName), ] = mappingPathSubset(fieldLabels); const isAnyRank = databaseTableOrRankName === formatTreeRank(anyTreeRank); @@ -164,19 +168,20 @@ export function generateMappingPathPreview( const tableNameFormatted = tablesToHide.has(databaseTableOrRankName) && databaseFieldName !== formattedEntry - ? [parentTableName || tableNameNonEmpty] + ? [parentTableOrTreeName || tableNameNonEmpty] : genericTables.has(databaseTableOrRankName) - ? [parentTableName, tableNameNonEmpty] + ? [parentTableOrTreeName, tableNameNonEmpty] : [tableNameNonEmpty]; return filterArray([ ...(valueIsTreeRank(databaseTableOrRankName) - ? [isAnyRank ? parentTableName : tableOrRankName] + ? [isAnyRank ? parentTableOrTreeName : tableOrRankName] : tableNameFormatted), fieldNameFormatted, ...(valueIsTreeRank(databaseTableOrRankName) && + valueIsTreeDefinition(databaseParentTableOrTreeName) && duplicateRanks?.has(databaseTableOrRankName) - ? [parentTableName] + ? [parentTableOrTreeName] : []), toManyIndexFormatted, ]) From 833adb5ec3b53cf0b4e45a8727c286e38c229f19 Mon Sep 17 00:00:00 2001 From: Sharad S Date: Thu, 5 Sep 2024 15:33:03 -0400 Subject: [PATCH 57/78] Fix upload plan when there are multiple ranks of the same tree in the mapper --- .../WbPlanView/uploadPlanBuilder.ts | 51 +++++++++++-------- 1 file changed, 30 insertions(+), 21 deletions(-) diff --git a/specifyweb/frontend/js_src/lib/components/WbPlanView/uploadPlanBuilder.ts b/specifyweb/frontend/js_src/lib/components/WbPlanView/uploadPlanBuilder.ts index 493933674dd..5137ebb6b25 100644 --- a/specifyweb/frontend/js_src/lib/components/WbPlanView/uploadPlanBuilder.ts +++ b/specifyweb/frontend/js_src/lib/components/WbPlanView/uploadPlanBuilder.ts @@ -52,33 +52,42 @@ const toTreeRecordVariety = (lines: RA): TreeRecord => { return { ranks: Object.fromEntries( - indexMappings(lines).map(([fullName, rankMappedFields]) => { - const rankName = valueIsTreeRank(fullName) - ? getNameFromTreeRankName(fullName) - : getNameFromTreeRankName(rankMappedFields[0].mappingPath[0]); + indexMappings(lines).flatMap(([fullName, rankMappedFields]) => { + // For collections with only 1 tree, rankMappedFields already has the correct format for generating an upload plan + // When there are multiple trees in the mapper, rankMappedFields is actually a mapping path of trees mapped to ranks + // which means we need to get the index mappings of it again to get the actual rankMappedFields + const rankMappings: RA]> = + valueIsTreeRank(fullName) + ? [[fullName, rankMappedFields]] + : indexMappings(rankMappedFields); const treeName = valueIsTreeDefinition(fullName) ? getNameFromTreeDefinitionName(fullName) : undefined; - const treeId = - treeName === undefined - ? treeDefinitions.find(({ ranks }) => - ranks.find((r) => r.name === rankName) - )?.definition.id - : treeDefinitions.find( - ({ definition }) => definition.name === treeName - )?.definition.id; + return rankMappings.map(([fullName, mappedFields]) => { + const rankName = getNameFromTreeRankName(fullName); - return [ - treeId === undefined - ? rankName - : formatTreeRankWithTreeId(rankName, treeId), - { - treeNodeCols: toTreeRecordRanks(rankMappedFields), - treeId, - }, - ]; + // Resolve treeId using treeName or using rankName as fallback + const treeId = + treeName === undefined + ? treeDefinitions.find(({ ranks }) => + ranks.find((r) => r.name === rankName) + )?.definition.id + : treeDefinitions.find( + ({ definition }) => definition.name === treeName + )?.definition.id; + + return [ + treeId === undefined + ? rankName + : formatTreeRankWithTreeId(rankName, treeId), + { + treeNodeCols: toTreeRecordRanks(mappedFields), + treeId, + }, + ]; + }); }) ), }; From bc3e9f6d392145248b6d1e52532b2c5b6024298c Mon Sep 17 00:00:00 2001 From: Sharad S Date: Thu, 5 Sep 2024 19:36:56 +0000 Subject: [PATCH 58/78] Lint code with ESLint and Prettier Triggered by dd7e3f82080ae6e66080fb1b496c7da87dcda12c on branch refs/heads/issue-4980 --- .../lib/components/WbPlanView/uploadPlanBuilder.ts | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/specifyweb/frontend/js_src/lib/components/WbPlanView/uploadPlanBuilder.ts b/specifyweb/frontend/js_src/lib/components/WbPlanView/uploadPlanBuilder.ts index 5137ebb6b25..0fbc2d45f3b 100644 --- a/specifyweb/frontend/js_src/lib/components/WbPlanView/uploadPlanBuilder.ts +++ b/specifyweb/frontend/js_src/lib/components/WbPlanView/uploadPlanBuilder.ts @@ -5,7 +5,8 @@ import { strictGetTable } from '../DataModel/tables'; import type { Tables } from '../DataModel/types'; import { getTreeDefinitions, isTreeTable } from '../InitialContext/treeRanks'; import { defaultColumnOptions } from './linesGetter'; -import { formatTreeRankWithTreeId, SplitMappingPath } from './mappingHelpers'; +import type { SplitMappingPath } from './mappingHelpers'; +import { formatTreeRankWithTreeId } from './mappingHelpers'; import { getNameFromTreeDefinitionName, valueIsTreeDefinition, @@ -53,9 +54,11 @@ const toTreeRecordVariety = (lines: RA): TreeRecord => { return { ranks: Object.fromEntries( indexMappings(lines).flatMap(([fullName, rankMappedFields]) => { - // For collections with only 1 tree, rankMappedFields already has the correct format for generating an upload plan - // When there are multiple trees in the mapper, rankMappedFields is actually a mapping path of trees mapped to ranks - // which means we need to get the index mappings of it again to get the actual rankMappedFields + /* + * For collections with only 1 tree, rankMappedFields already has the correct format for generating an upload plan + * When there are multiple trees in the mapper, rankMappedFields is actually a mapping path of trees mapped to ranks + * which means we need to get the index mappings of it again to get the actual rankMappedFields + */ const rankMappings: RA]> = valueIsTreeRank(fullName) ? [[fullName, rankMappedFields]] From 16671baab00df3426d40ec5bad29613eb5339a35 Mon Sep 17 00:00:00 2001 From: alec_dev Date: Thu, 5 Sep 2024 16:05:12 -0500 Subject: [PATCH 59/78] update treerecord _to_match function and remove rescope_tree_from_row_old --- specifyweb/workbench/upload/treerecord.py | 116 ++++------------------ 1 file changed, 20 insertions(+), 96 deletions(-) diff --git a/specifyweb/workbench/upload/treerecord.py b/specifyweb/workbench/upload/treerecord.py index 9604f0a0bc6..1fcf3b390eb 100644 --- a/specifyweb/workbench/upload/treerecord.py +++ b/specifyweb/workbench/upload/treerecord.py @@ -174,100 +174,9 @@ def disambiguate(self, disambiguation: DA) -> "ScopedTreeRecord": def get_treedefs(self) -> Set: # return set([self.treedef]) # old way - tree_def_model = get_treedef_model(self.name) treedefids = set([tree_rank_record.treedef_id for tree_rank_record in self.ranks.keys()]) - # return set([tree_rank_record.treedef_id for tree_rank_record in self.ranks.keys()]) - # return tree_rank_model.objects.filter(treedef_id__in=treedefids) return set(tree_def_model.objects.filter(id__in=treedefids)) - - # Remove this old version of the funciton - def rescope_tree_from_row_old(self, row: Row) -> Tuple["ScopedTreeRecord", Optional["WorkBenchParseFailure"]]: - def fetch_tree_node_model(name: str): - return getattr(models, name.lower().title()) - - def fetch_treedefitem_model(name: str): - return get_treedefitem_model(name) - - def get_ranks_mapping(ranks): - return {rank.rank_name: rank.treedef_id for rank in ranks.keys()} - - def find_non_null_ranks_in_row(row, rank_names): - return { - col - for col, value in row.items() - if value is not None and value != "" and col in rank_names - } - - def handle_no_ranks_in_row(): - if self.treedef is not None: - return self, None - first_rank = next(iter(self.ranks.keys())) - target_rank_treedef_id = first_rank.treedef_id - target_rank_treedef = tree_node_model.objects.filter( - definition_id=target_rank_treedef_id, parent=None - ).first() - treedefitems = list( - tree_rank_model.objects.filter( - treedef_id=target_rank_treedef_id - ).order_by("rankid") - ) - root = tree_node_model.objects.filter(definition_id=target_rank_treedef_id, parent=None).first() - return self._replace(treedef=target_rank_treedef, treedefitems=treedefitems, root=root), None - - def handle_multiple_ranks_in_row(ranks_in_row_not_null): - return self, WorkBenchParseFailure('multipleRanksInRow', {}, list(ranks_in_row_not_null)[0]) - - def handle_invalid_rank(target_rank_name): - return self, WorkBenchParseFailure('invalidRank', {'rank': target_rank_name}, target_rank_name) - - # def handle_extended_rank_name(rank_col_name): - # extracted_rank_name = None - # if ' - ' in rank_col_name: - # extracted_rank_name, treedef_name = rank_col_name.split(' - ') - # elif '~>' in rank_col_name: - # extracted_rank_name, treedef_name = rank_col_name.split('~>') - # if extracted_rank_name in set([rank.rank_name for rank in self.ranks]): - # return extracted_rank_name - # return rank_col_name - - # def modify_row_for_extended_rank_names(row): - # row_modified = {} - # for col_name in row.keys(): - # col_name = handle_extended_rank_name(col_name) - # row_modified[col_name] = row[col_name] - # return row_modified - - tree_rank_model = fetch_treedefitem_model(self.name) - tree_node_model = fetch_tree_node_model(self.name) - - if len(set(tr.treedef_id for tr in self.ranks.keys())) == 1: - return self, None - - # mod_row = modify_row_for_extended_rank_names(row) - ranks = get_ranks_mapping(self.ranks) - rank_names = set(ranks.keys()) - ranks_in_row_not_null = find_non_null_ranks_in_row(row, rank_names) - - if len(ranks_in_row_not_null) > 1: - return handle_multiple_ranks_in_row(ranks_in_row_not_null) - if not ranks_in_row_not_null: - return handle_no_ranks_in_row() - - target_rank_name = ranks_in_row_not_null.pop() - target_rank_treedef_id = ranks[target_rank_name] - treedefitem_query = tree_rank_model.objects.filter( - name=target_rank_name, treedef_id=target_rank_treedef_id - ).order_by("rankid") - - if not treedefitem_query.exists() or treedefitem_query.count() != 1: - return handle_invalid_rank(target_rank_name) - - treedefitems = list(tree_rank_model.objects.filter(treedef_id=target_rank_treedef_id).order_by("rankid")) - root = tree_node_model.objects.filter(definition_id=target_rank_treedef_id, parent=None).first() - target_rank_treedef = treedefitem_query.first().treedef - - return self._replace(treedef=target_rank_treedef, treedefitems=treedefitems, root=root), None def _get_not_null_ranks_columns_in_row(self, row: Row) -> List[TreeRankCell]: # Get rank columns that are not null in the row @@ -347,7 +256,7 @@ def bind( cache: Optional[Dict] = None, row_index: Optional[int] = None, ) -> Union["BoundTreeRecord", ParseFailures]: - parsedFields: Dict[str, List[ParseResult]] = {} + parsedFields: Dict[TreeRankRecord, List[ParseResult]] = {} parseFails: List[WorkBenchParseFailure] = [] rescoped_tree_record, parse_fail = self.rescope_tree_from_row(row) @@ -357,7 +266,7 @@ def bind( for tree_rank_record, cols in self.ranks.items(): nameColumn = cols['name'] presults, pfails = parse_many(collection, self.name, cols, row) - parsedFields[tree_rank_record.rank_name] = presults + parsedFields[tree_rank_record] = presults parseFails += pfails filters = {k: v for result in presults for k, v in result.filter_on.items()} if filters.get('name', None) is None: @@ -409,7 +318,7 @@ class BoundTreeRecord(NamedTuple): treedef: Any treedefitems: List root: Optional[Any] - parsedFields: Dict[str, List[ParseResult]] + parsedFields: Dict[TreeRankRecord, List[ParseResult]] uploadingAgentId: Optional[int] auditor: Auditor cache: Optional[Dict] @@ -452,10 +361,25 @@ def _handle_row(self, must_match: bool) -> UploadResult: return UploadResult(match_result, {}, {}) def _to_match(self) -> List[TreeDefItemWithParseResults]: + def has_non_null_values(parse_results): + return any( + value is not None + for result in parse_results + for value in result.filter_on.values() + ) + + def get_parse_results(tdi): + tree_rank_record = TreeRankRecord(tdi.name, tdi.treedef_id) + return self.parsedFields.get(tree_rank_record, []) + + def is_valid_tdi(tdi): + tree_rank_names = {trr.rank_name for trr in self.parsedFields.keys()} + return tdi.name in tree_rank_names and has_non_null_values(get_parse_results(tdi)) + return [ - TreeDefItemWithParseResults(tdi, self.parsedFields[tdi.name]) + TreeDefItemWithParseResults(tdi, get_parse_results(tdi)) for tdi in self.treedefitems - if tdi.name in self.parsedFields and any(v is not None for r in self.parsedFields[tdi.name] for v in r.filter_on.values()) + if is_valid_tdi(tdi) ] def _match(self, tdiwprs: List[TreeDefItemWithParseResults]) -> Tuple[List[TreeDefItemWithParseResults], MatchResult]: From 1a368d94d54ffff0ef3c60c7182572712b996639 Mon Sep 17 00:00:00 2001 From: alec_dev Date: Thu, 5 Sep 2024 16:14:39 -0500 Subject: [PATCH 60/78] remove fullname hack --- specifyweb/workbench/upload/treerecord.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/specifyweb/workbench/upload/treerecord.py b/specifyweb/workbench/upload/treerecord.py index 1fcf3b390eb..bcc6d3e92e8 100644 --- a/specifyweb/workbench/upload/treerecord.py +++ b/specifyweb/workbench/upload/treerecord.py @@ -563,8 +563,6 @@ def _upload(self, to_upload: List[TreeDefItemWithParseResults], matched: Union[M def _do_insert(self, model, **kwargs): obj = model(**kwargs) - if hasattr(obj, 'fullname') and obj.fullname is None: - obj.fullname = obj.name obj.save(skip_tree_extras=True) return obj From c84860fa656dcc71e66a395673f80d349965df0d Mon Sep 17 00:00:00 2001 From: Sharad S Date: Fri, 6 Sep 2024 10:02:23 -0400 Subject: [PATCH 61/78] Use tree name in header always - note: will not be added to dbs with only 1 tree --- .../js_src/lib/components/WbPlanView/headerHelper.ts | 12 +----------- .../lib/components/WbPlanView/mappingPreview.ts | 6 ++---- 2 files changed, 3 insertions(+), 15 deletions(-) diff --git a/specifyweb/frontend/js_src/lib/components/WbPlanView/headerHelper.ts b/specifyweb/frontend/js_src/lib/components/WbPlanView/headerHelper.ts index ab23c77f7a4..97915632fd8 100644 --- a/specifyweb/frontend/js_src/lib/components/WbPlanView/headerHelper.ts +++ b/specifyweb/frontend/js_src/lib/components/WbPlanView/headerHelper.ts @@ -9,7 +9,6 @@ import type { RA } from '../../utils/types'; import { getUniqueName } from '../../utils/uniquifyName'; import type { Tables } from '../DataModel/types'; import type { MappingLine } from './Mapper'; -import { valueIsTreeRank } from './mappingHelpers'; import { generateMappingPathPreview } from './mappingPreview'; export function uniquifyHeaders( @@ -35,22 +34,13 @@ export function renameNewlyCreatedHeaders( headers: RA, lines: RA ): RA { - const duplicateRanks = new Set( - lines - .flatMap(({ mappingPath }) => mappingPath) - .filter( - (rank, index, ranks) => - valueIsTreeRank(rank) && ranks.indexOf(rank) !== index - ) - ); - const generatedHeaderPreviews = Object.fromEntries( lines .map((line, index) => ({ line, index })) .filter(({ line }) => !headers.includes(line.headerName)) .map(({ line, index }) => [ index, - generateMappingPathPreview(baseTableName, line.mappingPath, duplicateRanks), + generateMappingPathPreview(baseTableName, line.mappingPath), ]) ); diff --git a/specifyweb/frontend/js_src/lib/components/WbPlanView/mappingPreview.ts b/specifyweb/frontend/js_src/lib/components/WbPlanView/mappingPreview.ts index 0c4e05a4d34..ad03b945177 100644 --- a/specifyweb/frontend/js_src/lib/components/WbPlanView/mappingPreview.ts +++ b/specifyweb/frontend/js_src/lib/components/WbPlanView/mappingPreview.ts @@ -84,8 +84,7 @@ const mappingPathSubset = ( */ export function generateMappingPathPreview( baseTableName: keyof Tables, - mappingPath: MappingPath, - duplicateRanks?: ReadonlySet + mappingPath: MappingPath ): string { if (mappingPath.length === 0) return strictGetTable(baseTableName).label; @@ -179,8 +178,7 @@ export function generateMappingPathPreview( : tableNameFormatted), fieldNameFormatted, ...(valueIsTreeRank(databaseTableOrRankName) && - valueIsTreeDefinition(databaseParentTableOrTreeName) && - duplicateRanks?.has(databaseTableOrRankName) + valueIsTreeDefinition(databaseParentTableOrTreeName) ? [parentTableOrTreeName] : []), toManyIndexFormatted, From 67f874ecb4daddca2147d216e7c0d2fdd5eb1d17 Mon Sep 17 00:00:00 2001 From: alec_dev Date: Fri, 6 Sep 2024 09:22:24 -0500 Subject: [PATCH 62/78] add back WorkBenchParseFailure for multipleRanksInRow and noRoot --- specifyweb/workbench/upload/treerecord.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/specifyweb/workbench/upload/treerecord.py b/specifyweb/workbench/upload/treerecord.py index bcc6d3e92e8..521ee62f71b 100644 --- a/specifyweb/workbench/upload/treerecord.py +++ b/specifyweb/workbench/upload/treerecord.py @@ -231,8 +231,8 @@ def rescope_tree_from_row(self, row: Row) -> Tuple["ScopedTreeRecord", Optional[ # return self, WorkBenchParseFailure('noRanksInRow', {}, None) return self, None elif len(targeted_treedefids) > 1: - # return self, WorkBenchParseFailure('multipleRanksInRow', {}, list(ranks_in_row_not_null)[0]) logger.warning(f"Multiple treedefs found in row: {targeted_treedefids}") + return self, WorkBenchParseFailure('multipleRanksInRow', {}, list(ranks_columns_in_row_not_null)[0]) target_rank_treedef_id = targeted_treedefids.pop() target_rank_treedef = tree_def_model.objects.get(id=target_rank_treedef_id) @@ -243,6 +243,9 @@ def rescope_tree_from_row(self, row: Row) -> Tuple["ScopedTreeRecord", Optional[ # Based on the target treedef, get the treedefitems and root for the tree treedefitems = list(tree_rank_model.objects.filter(treedef_id=target_rank_treedef_id).order_by("rankid")) root = tree_node_model.objects.filter(definition_id=target_rank_treedef_id, parent=None).first() + if root is None: + logger.warning(f"No root found for treedef {target_rank_treedef_id}") + return self, WorkBenchParseFailure('noRoot', {}, None) # Return a new ScopedTreeRecord with the correct treedefitem and treedef return self._replace(treedef=target_rank_treedef, treedefitems=treedefitems, root=root), None From 3b7557837ea3b87dfd94a6db30d08ebf2a4f22a6 Mon Sep 17 00:00:00 2001 From: alec_dev Date: Fri, 6 Sep 2024 12:13:20 -0500 Subject: [PATCH 63/78] functional refactoring --- .../workbench/upload/tests/example_plan.py | 1 - specifyweb/workbench/upload/treerecord.py | 297 +++++++++++++----- .../workbench/upload/upload_plan_schema.py | 47 ++- 3 files changed, 251 insertions(+), 94 deletions(-) diff --git a/specifyweb/workbench/upload/tests/example_plan.py b/specifyweb/workbench/upload/tests/example_plan.py index 9d2d99433af..7aabb873bcb 100644 --- a/specifyweb/workbench/upload/tests/example_plan.py +++ b/specifyweb/workbench/upload/tests/example_plan.py @@ -2,7 +2,6 @@ from ..tomany import ToManyRecord from ..treerecord import TreeRecord from ..upload_plan_schema import parse_column_options -# from specifyweb.specify import models as spmodels json = dict( baseTableName = 'Collectionobject', diff --git a/specifyweb/workbench/upload/treerecord.py b/specifyweb/workbench/upload/treerecord.py index 521ee62f71b..da71a54b9f2 100644 --- a/specifyweb/workbench/upload/treerecord.py +++ b/specifyweb/workbench/upload/treerecord.py @@ -41,23 +41,56 @@ def create( base_treedef_id: Optional[int] = None, ) -> 'TreeRank': tree_model = get_treedefitem_model(tree) - tree_lower = tree.lower() def _build_filter_kwargs(rank_name: str, treedef_id: Optional[int] = None) -> Dict[str, Any]: - filter_kwargs = {'name': rank_name} - if treedef_id is not None: - filter_kwargs['treedef_id'] = treedef_id # type: ignore + """ + Build filter keyword arguments for querying treedef items. + """ + + # Create base keyword arguments for filtering + def create_base_kwargs(rank_name: str) -> Dict[str, Any]: + return {'name': rank_name} + + # Add treedef ID to keyword arguments if present + def add_treedef_id_if_present(kwargs: Dict[str, Any], treedef_id: Optional[int]) -> Dict[str, Any]: + if treedef_id is not None: + kwargs['treedef_id'] = treedef_id + return kwargs + + filter_kwargs = create_base_kwargs(rank_name) + filter_kwargs = add_treedef_id_if_present(filter_kwargs, treedef_id) return filter_kwargs def _filter_by_base_treedef_id(treedefitems, rank_name: str, base_treedef_id: Optional[int]): - if base_treedef_id is None: - raise ValueError(f"Multiple treedefitems found for rank {rank_name}") - treedefitems = treedefitems.filter(treedef_id=base_treedef_id) - if not treedefitems.exists(): - raise ValueError(f"No treedefitems found for rank {rank_name} and base treedef {base_treedef_id}") - if treedefitems.count() > 1: - raise ValueError(f"Multiple treedefitems found for rank {rank_name} and base treedef {base_treedef_id}") - return treedefitems + """ + Filter treedefitems by base_treedef_id and ensure only one item is found. + """ + + # Check if base treedef ID is present + def check_base_treedef_id(base_treedef_id): + if base_treedef_id is None: + raise ValueError(f"Multiple treedefitems found for rank {rank_name}") + + # Filter treedefitems by base treedef ID + def filter_items_by_treedef_id(treedefitems, base_treedef_id): + return treedefitems.filter(treedef_id=base_treedef_id) + + # Validate filtered items + def validate_filtered_items(treedefitems): + if not treedefitems.exists(): + raise ValueError( + f"No treedefitems found for rank {rank_name} and base treedef {base_treedef_id}" + ) + if treedefitems.count() > 1: + raise ValueError( + f"Multiple treedefitems found for rank {rank_name} and base treedef {base_treedef_id}" + ) + + # Check base treedef ID and filter treedefitems, then validate + check_base_treedef_id(base_treedef_id) + filtered_items = filter_items_by_treedef_id(treedefitems, base_treedef_id) + validate_filtered_items(filtered_items) + return filtered_items def _get_treedef_id( rank_name: str, @@ -65,40 +98,81 @@ def _get_treedef_id( treedef_id: Optional[int], base_treedef_id: Optional[int], ) -> int: - filter_kwargs = _build_filter_kwargs(rank_name, treedef_id) - treedefitems = tree_model.objects.filter(**filter_kwargs) + """ + Get the treedef ID for the given rank name and tree. + """ + # Fetch treedefitems based on filter keyword arguments + def fetch_treedefitems(filter_kwargs): + return tree_model.objects.filter(**filter_kwargs) + + # Get the first item from the queryset + def get_first_item(treedefitems): + first_item = treedefitems.first() + if first_item is None: + raise ValueError(f"No treedefitems found for rank {rank_name}") + return first_item + + # Handle cases where multiple items are found + def handle_multiple_items(treedefitems): + if treedefitems.count() > 1: + return _filter_by_base_treedef_id(treedefitems, rank_name, base_treedef_id) + return treedefitems + + # Build filter keyword arguments and fetch treedefitems + filter_kwargs = _build_filter_kwargs(rank_name, treedef_id) + treedefitems = fetch_treedefitems(filter_kwargs) + + # Handle cases where no items are found if not treedefitems.exists() and treedef_id is not None: filter_kwargs = _build_filter_kwargs(rank_name) - treedefitems = tree_model.objects.filter(**filter_kwargs) - - if treedefitems.count() > 1: - treedefitems = _filter_by_base_treedef_id(treedefitems, rank_name, base_treedef_id) + treedefitems = fetch_treedefitems(filter_kwargs) - first_item = treedefitems.first() - if first_item is None: - raise ValueError(f"No treedefitems found for rank {rank_name}") + treedefitems = handle_multiple_items(treedefitems) + first_item = get_first_item(treedefitems) return first_item.treedef_id - if re.match(r'.*~>\d+$', rank_name): - rank_name, treedef_id_str = re.split(r'~>', rank_name) - treedef_id = _get_treedef_id(rank_name, tree, int(treedef_id_str), base_treedef_id) - else: - treedef_id = _get_treedef_id(rank_name, tree, treedef_id, base_treedef_id) - - return TreeRank(rank_name, treedef_id, tree_lower) + # Get the treedef_id using the helper function + final_treedef_id = _get_treedef_id(rank_name, tree, treedef_id, base_treedef_id) + + # Return an instance of TreeRank + return TreeRank(rank_name=rank_name, treedef_id=final_treedef_id, tree=tree) def check_rank(self) -> bool: - tree_model = get_treedefitem_model(self.tree) - rank = tree_model.objects.filter(name=self.rank_name, treedef_id=self.treedef_id) - return rank.exists() and rank.count() == 1 + """ + Check if the rank exists and is unique for the given tree and treedef ID. + """ + + # Get the tree model + def get_tree_model(tree: str): + return get_treedefitem_model(tree) + + # Filter the rank by name + def filter_rank(tree_model, rank_name: str, treedef_id: int): + return tree_model.objects.filter(name=rank_name, treedef_id=treedef_id) + + # Check if the rank exists and is unique + def rank_exists_and_unique(rank): + return rank.exists() and rank.count() == 1 + + tree_model = get_tree_model(self.tree) + rank = filter_rank(tree_model, self.rank_name, self.treedef_id) + return rank_exists_and_unique(rank) def validate_rank(self) -> None: + """ + Validate the rank for the given tree and treedef ID. + """ + if not self.check_rank(): raise ValueError(f"Invalid rank {self.rank_name} for treedef {self.treedef_id}") def tree_rank_record(self) -> 'TreeRankRecord': + """ + Create a TreeRankRecord instance. + """ + assert self.rank_name is not None, "Rank name is required" assert self.treedef_id is not None, "Treedef ID is required" assert self.tree is not None and self.tree.lower() in SPECIFY_TREES, "Tree is required" @@ -114,18 +188,22 @@ class TreeRankRecord(NamedTuple): rank_name: str treedef_id: int + # Create a TreeRankRecord instance def to_json(self) -> Dict: return { "rank": self.rank_name, "treedefId": self.treedef_id, } + # Get the key for the TreeRankRecord instance def to_key(self) -> Tuple[str, int]: return (self.rank_name, self.treedef_id) + # Check if the rank exists and is unique for the given tree and treedef ID def check_rank(self, tree: str) -> bool: return TreeRank.create(self.rank_name, tree, self.treedef_id).check_rank() + # Validate the rank for the given tree and treedef ID def validate_rank(self, tree) -> None: TreeRank.create(self.rank_name, tree, self.treedef_id).validate_rank() @@ -179,25 +257,47 @@ def get_treedefs(self) -> Set: return set(tree_def_model.objects.filter(id__in=treedefids)) def _get_not_null_ranks_columns_in_row(self, row: Row) -> List[TreeRankCell]: - # Get rank columns that are not null in the row - ranks_columns_in_row_not_null = [] - for row_key, row_value in row.items(): - if not row_value: - continue - for tree_rank_record, cols in self.ranks.items(): - for col_name, col_opts in cols.items(): - formatted_column = f'{col_opts.column} - {col_name}' if col_name != 'name' else col_opts.column - if formatted_column == row_key: - ranks_columns_in_row_not_null.append( - TreeRankCell( - tree_rank_record.treedef_id, # treedefid - tree_rank_record.rank_name, # treedefitem_name - col_name, # tree_node_attribute - row_value, # upload_value - ) - ) - break - return ranks_columns_in_row_not_null + """ + Get rank columns that are not null in the row. + """ + + # Check if the value is not null + def is_not_null(value: Any) -> bool: + return bool(value) + + # Format the column name + def format_column(col_name: str, col_opts: Any) -> str: + return f'{col_opts.column} - {col_name}' if col_name != 'name' else col_opts.column + + # Match the column + def match_column(row_key: str, formatted_column: str) -> bool: + return formatted_column == row_key + + # Create a TreeRankCell instance + def create_tree_rank_cell(tree_rank_record: Any, col_name: str, row_value: Any) -> TreeRankCell: + return TreeRankCell( + tree_rank_record.treedef_id, # treedefid + tree_rank_record.rank_name, # treedefitem_name + col_name, # tree_node_attribute + row_value, # upload_value + ) + + # Process the row item + def process_row_item(row_key: str, row_value: Any) -> List[TreeRankCell]: + if not is_not_null(row_value): + return [] + return [ + create_tree_rank_cell(tree_rank_record, col_name, row_value) + for tree_rank_record, cols in self.ranks.items() + for col_name, col_opts in cols.items() + if match_column(row_key, format_column(col_name, col_opts)) + ] + + return [ + cell + for row_key, row_value in row.items() + for cell in process_row_item(row_key, row_value) + ] def _filter_target_rank_columns(self, ranks_columns_in_row_not_null, target_rank_treedef_id) -> List[TreeRankCell]: # Filter ranks_columns_in_row_not_null to only include columns that are part of the target treedef @@ -209,45 +309,81 @@ def _filter_target_rank_columns(self, ranks_columns_in_row_not_null, target_rank ) def rescope_tree_from_row(self, row: Row) -> Tuple["ScopedTreeRecord", Optional["WorkBenchParseFailure"]]: - # Starting out with a row with tree data. - # The row can have multiple tree ranks, and each rank can be associated with multiple columns, like name, authoer, etc. - # First need to match each rank column to a treedefitem. - # Second need to narrow down to the target rank based on which values in the row are not null. - # Lastly, need to return a new ScopedTreeRecord with the correct treedef, treedefitems, and root for this row. - - tree_def_model = get_treedef_model(self.name) - tree_rank_model = get_treedefitem_model(self.name) - tree_node_model = getattr(models, self.name.lower().title()) + """Rescope tree from row data.""" - if len(set(tr.treedef_id for tr in self.ranks.keys())) == 1: - return self, None + # Get models based on the name + def get_models(name: str): + tree_def_model = get_treedef_model(name) + tree_rank_model = get_treedefitem_model(name) + tree_node_model = getattr(models, name.lower().title()) + return tree_def_model, tree_rank_model, tree_node_model # Get rank columns that are not null in the row - ranks_columns_in_row_not_null: List[TreeRankCell] = self._get_not_null_ranks_columns_in_row(row) + def get_not_null_ranks_columns(row: Row) -> List[TreeRankCell]: + return self._get_not_null_ranks_columns_in_row(row) # Determine the target treedef based on the columns that are not null - targeted_treedefids = set([rank_column.treedef_id for rank_column in ranks_columns_in_row_not_null]) - if targeted_treedefids is None or len(targeted_treedefids) == 0: - # return self, WorkBenchParseFailure('noRanksInRow', {}, None) + def get_targeted_treedefids(ranks_columns: List[TreeRankCell]) -> Set[int]: + return set(rank_column.treedef_id for rank_column in ranks_columns) + + # Handle cases where there are multiple or no treedefs + def handle_multiple_or_no_treedefs(targeted_treedefids: Set[int], ranks_columns: List[TreeRankCell]): + if not targeted_treedefids: + return self, None + elif len(targeted_treedefids) > 1: + logger.warning(f"Multiple treedefs found in row: {targeted_treedefids}") + return self, WorkBenchParseFailure('multipleRanksInRow', {}, ranks_columns[0]) + return None + + # Retrieve the target rank treedef + def get_target_rank_treedef(tree_def_model, target_rank_treedef_id: int): + return tree_def_model.objects.get(id=target_rank_treedef_id) + + # Retrieve the treedef items and root for the tree + def get_treedefitems_and_root(tree_rank_model, tree_node_model, target_rank_treedef_id: int): + # Fetch treedef items + def fetch_treedefitems(): + return list(tree_rank_model.objects.filter(treedef_id=target_rank_treedef_id).order_by("rankid")) + + # Fetch root node + def fetch_root(): + return tree_node_model.objects.filter(definition_id=target_rank_treedef_id, parent=None).first() + + # Check if root is None and log warning + def check_root(root): + if root is None: + logger.warning(f"No root found for treedef {target_rank_treedef_id}") + return None, WorkBenchParseFailure('noRoot', {}, None) + return root + + treedefitems = fetch_treedefitems() + root = fetch_root() + root_checked = check_root(root) + + if root_checked is not root: + return root_checked + + return treedefitems, root + + tree_def_model, tree_rank_model, tree_node_model = get_models(self.name) + + if len(set(tr.treedef_id for tr in self.ranks.keys())) == 1: return self, None - elif len(targeted_treedefids) > 1: - logger.warning(f"Multiple treedefs found in row: {targeted_treedefids}") - return self, WorkBenchParseFailure('multipleRanksInRow', {}, list(ranks_columns_in_row_not_null)[0]) - - target_rank_treedef_id = targeted_treedefids.pop() - target_rank_treedef = tree_def_model.objects.get(id=target_rank_treedef_id) - # Filter ranks_columns_in_row_not_null to only include columns that are part of the target treedef - # ranks_columns = self._filter_target_rank_columns(ranks_columns_in_row_not_null, target_rank_treedef_id) + ranks_columns_in_row_not_null = get_not_null_ranks_columns(row) + targeted_treedefids = get_targeted_treedefids(ranks_columns_in_row_not_null) - # Based on the target treedef, get the treedefitems and root for the tree - treedefitems = list(tree_rank_model.objects.filter(treedef_id=target_rank_treedef_id).order_by("rankid")) - root = tree_node_model.objects.filter(definition_id=target_rank_treedef_id, parent=None).first() + result = handle_multiple_or_no_treedefs(targeted_treedefids, ranks_columns_in_row_not_null) + if result: + return result + + target_rank_treedef_id = targeted_treedefids.pop() + target_rank_treedef = get_target_rank_treedef(tree_def_model, target_rank_treedef_id) + + treedefitems, root = get_treedefitems_and_root(tree_rank_model, tree_node_model, target_rank_treedef_id) if root is None: - logger.warning(f"No root found for treedef {target_rank_treedef_id}") return self, WorkBenchParseFailure('noRoot', {}, None) - # Return a new ScopedTreeRecord with the correct treedefitem and treedef return self._replace(treedef=target_rank_treedef, treedefitems=treedefitems, root=root), None def bind( @@ -364,6 +500,8 @@ def _handle_row(self, must_match: bool) -> UploadResult: return UploadResult(match_result, {}, {}) def _to_match(self) -> List[TreeDefItemWithParseResults]: + + # Check if the parse results have non-null values def has_non_null_values(parse_results): return any( value is not None @@ -371,14 +509,17 @@ def has_non_null_values(parse_results): for value in result.filter_on.values() ) + # Get parse results for a TDI def get_parse_results(tdi): tree_rank_record = TreeRankRecord(tdi.name, tdi.treedef_id) return self.parsedFields.get(tree_rank_record, []) + # Check if the TDI is valid def is_valid_tdi(tdi): tree_rank_names = {trr.rank_name for trr in self.parsedFields.keys()} return tdi.name in tree_rank_names and has_non_null_values(get_parse_results(tdi)) + # Create a TreeDefItemWithParseResults instance return [ TreeDefItemWithParseResults(tdi, get_parse_results(tdi)) for tdi in self.treedefitems diff --git a/specifyweb/workbench/upload/upload_plan_schema.py b/specifyweb/workbench/upload/upload_plan_schema.py index a9034b1f57d..997d0855c2a 100644 --- a/specifyweb/workbench/upload/upload_plan_schema.py +++ b/specifyweb/workbench/upload/upload_plan_schema.py @@ -1,3 +1,4 @@ +from functools import reduce from os import name from typing import Dict, Any, Optional, Union, Tuple import logging @@ -309,6 +310,9 @@ def defer_scope_upload_table(default_collection, table: Table, to_parse: Dict, d ) def parse_tree_record(collection, table: Table, to_parse: Dict, base_treedefid: Optional[int] = None) -> TreeRecord: + """Parse tree record from the given data""" + + # Parse rank details def parse_rank(name_or_cols): if isinstance(name_or_cols, str): return name_or_cols, {'name': parse_column_options(name_or_cols)}, None @@ -317,21 +321,34 @@ def parse_rank(name_or_cols): treedefid = name_or_cols.get('treeId') return rank_name, tree_node_cols, treedefid - ranks: Dict[Union[str, TreeRankRecord], Dict[str, ColumnOptions]] = {} - for rank, name_or_cols in to_parse['ranks'].items(): - rank_name, parsed_cols, treedefid = parse_rank(name_or_cols) - tree_rank_record = TreeRank.create(rank, table.name, treedefid, base_treedefid).tree_rank_record() - ranks[tree_rank_record] = parsed_cols - - # Update the schema with the treeId - # if not isinstance(name_or_cols, str): - # to_parse['ranks'][rank]['treeId'] = tree_rank_record.treedef_id - - if base_treedefid is None: - base_treedefid = tree_rank_record.treedef_id - - for rank, cols in ranks.items(): - assert 'name' in cols, to_parse + # Create tree rank record + def create_tree_rank_record(rank, table_name, treedefid, base_treedefid): + return TreeRank.create(rank, table_name, treedefid, base_treedefid).tree_rank_record() + + def parse_ranks(to_parse, table_name, base_treedefid): + def parse_single_rank(rank, name_or_cols, base_treedefid): + rank_name, parsed_cols, treedefid = parse_rank(name_or_cols) + tree_rank_record = create_tree_rank_record(rank, table_name, treedefid, base_treedefid) + return tree_rank_record, parsed_cols, tree_rank_record.treedef_id + + def aggregate_ranks(acc, item): + rank, name_or_cols = item + tree_rank_record, parsed_cols, treedefid = parse_single_rank(rank, name_or_cols, acc[1]) + acc[0][tree_rank_record] = parsed_cols + if acc[1] is None: + acc[1] = treedefid + return acc + + ranks, base_treedefid = reduce(aggregate_ranks, to_parse['ranks'].items(), ({}, base_treedefid)) + return ranks, base_treedefid + + # Validate ranks by checking if 'name' is present in each rank + def validate_ranks(ranks, to_parse): + for rank, cols in ranks.items(): + assert 'name' in cols, to_parse + + ranks, base_treedefid = parse_ranks(to_parse, table.name, base_treedefid) + validate_ranks(ranks, to_parse) return TreeRecord( name=table.django_name, From 2f1cfccd2b20f16d51545defb57cbb25002c2613 Mon Sep 17 00:00:00 2001 From: alec_dev Date: Fri, 6 Sep 2024 13:33:19 -0500 Subject: [PATCH 64/78] remove unneeded exception --- specifyweb/workbench/upload/tests/example_plan.py | 1 - specifyweb/workbench/upload/treerecord.py | 3 --- 2 files changed, 4 deletions(-) diff --git a/specifyweb/workbench/upload/tests/example_plan.py b/specifyweb/workbench/upload/tests/example_plan.py index 9d2d99433af..7aabb873bcb 100644 --- a/specifyweb/workbench/upload/tests/example_plan.py +++ b/specifyweb/workbench/upload/tests/example_plan.py @@ -2,7 +2,6 @@ from ..tomany import ToManyRecord from ..treerecord import TreeRecord from ..upload_plan_schema import parse_column_options -# from specifyweb.specify import models as spmodels json = dict( baseTableName = 'Collectionobject', diff --git a/specifyweb/workbench/upload/treerecord.py b/specifyweb/workbench/upload/treerecord.py index 521ee62f71b..fddedb55fe7 100644 --- a/specifyweb/workbench/upload/treerecord.py +++ b/specifyweb/workbench/upload/treerecord.py @@ -243,9 +243,6 @@ def rescope_tree_from_row(self, row: Row) -> Tuple["ScopedTreeRecord", Optional[ # Based on the target treedef, get the treedefitems and root for the tree treedefitems = list(tree_rank_model.objects.filter(treedef_id=target_rank_treedef_id).order_by("rankid")) root = tree_node_model.objects.filter(definition_id=target_rank_treedef_id, parent=None).first() - if root is None: - logger.warning(f"No root found for treedef {target_rank_treedef_id}") - return self, WorkBenchParseFailure('noRoot', {}, None) # Return a new ScopedTreeRecord with the correct treedefitem and treedef return self._replace(treedef=target_rank_treedef, treedefitems=treedefitems, root=root), None From 0432501450375f4f8b5f9e24870b712c6080b00e Mon Sep 17 00:00:00 2001 From: alec_dev Date: Fri, 6 Sep 2024 16:58:32 -0500 Subject: [PATCH 65/78] fix typecheck in WorkBenchParseFailure call --- specifyweb/workbench/upload/treerecord.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specifyweb/workbench/upload/treerecord.py b/specifyweb/workbench/upload/treerecord.py index fddedb55fe7..1c74724a6d7 100644 --- a/specifyweb/workbench/upload/treerecord.py +++ b/specifyweb/workbench/upload/treerecord.py @@ -232,7 +232,7 @@ def rescope_tree_from_row(self, row: Row) -> Tuple["ScopedTreeRecord", Optional[ return self, None elif len(targeted_treedefids) > 1: logger.warning(f"Multiple treedefs found in row: {targeted_treedefids}") - return self, WorkBenchParseFailure('multipleRanksInRow', {}, list(ranks_columns_in_row_not_null)[0]) + return self, WorkBenchParseFailure('multipleRanksInRow', {}, list(ranks_columns_in_row_not_null)[0].treedefitem_name) target_rank_treedef_id = targeted_treedefids.pop() target_rank_treedef = tree_def_model.objects.get(id=target_rank_treedef_id) From 8a2ba88fd64ba6d510cc3d89441fe393f4f56496 Mon Sep 17 00:00:00 2001 From: alec_dev Date: Fri, 6 Sep 2024 17:25:43 -0500 Subject: [PATCH 66/78] formatting --- specifyweb/workbench/upload/treerecord.py | 39 +++++++++++++------ .../workbench/upload/upload_plan_schema.py | 4 +- 2 files changed, 28 insertions(+), 15 deletions(-) diff --git a/specifyweb/workbench/upload/treerecord.py b/specifyweb/workbench/upload/treerecord.py index d1b8753df3c..fce2ca17465 100644 --- a/specifyweb/workbench/upload/treerecord.py +++ b/specifyweb/workbench/upload/treerecord.py @@ -40,9 +40,12 @@ def create( treedef_id: Optional[int] = None, base_treedef_id: Optional[int] = None, ) -> 'TreeRank': + """ + Create a TreeRank instance with the given rank name, tree, and optional treedef IDs. + """ tree_model = get_treedefitem_model(tree) - def _build_filter_kwargs(rank_name: str, treedef_id: Optional[int] = None) -> Dict[str, Any]: + def build_filter_kwargs(rank_name: str, treedef_id: Optional[int] = None) -> Dict[str, Any]: """ Build filter keyword arguments for querying treedef items. """ @@ -61,7 +64,7 @@ def add_treedef_id_if_present(kwargs: Dict[str, Any], treedef_id: Optional[int]) filter_kwargs = add_treedef_id_if_present(filter_kwargs, treedef_id) return filter_kwargs - def _filter_by_base_treedef_id(treedefitems, rank_name: str, base_treedef_id: Optional[int]): + def filter_by_base_treedef_id(treedefitems, rank_name: str, base_treedef_id: Optional[int]): """ Filter treedefitems by base_treedef_id and ensure only one item is found. """ @@ -92,7 +95,7 @@ def validate_filtered_items(treedefitems): validate_filtered_items(filtered_items) return filtered_items - def _get_treedef_id( + def get_treedef_id( rank_name: str, tree: str, treedef_id: Optional[int], @@ -116,16 +119,16 @@ def get_first_item(treedefitems): # Handle cases where multiple items are found def handle_multiple_items(treedefitems): if treedefitems.count() > 1: - return _filter_by_base_treedef_id(treedefitems, rank_name, base_treedef_id) + return filter_by_base_treedef_id(treedefitems, rank_name, base_treedef_id) return treedefitems # Build filter keyword arguments and fetch treedefitems - filter_kwargs = _build_filter_kwargs(rank_name, treedef_id) + filter_kwargs = build_filter_kwargs(rank_name, treedef_id) treedefitems = fetch_treedefitems(filter_kwargs) # Handle cases where no items are found if not treedefitems.exists() and treedef_id is not None: - filter_kwargs = _build_filter_kwargs(rank_name) + filter_kwargs = build_filter_kwargs(rank_name) treedefitems = fetch_treedefitems(filter_kwargs) treedefitems = handle_multiple_items(treedefitems) @@ -133,11 +136,20 @@ def handle_multiple_items(treedefitems): return first_item.treedef_id - # Get the treedef_id using the helper function - final_treedef_id = _get_treedef_id(rank_name, tree, treedef_id, base_treedef_id) + def extract_treedef_id(rank_name: str) -> Tuple[Union[str, Any], Optional[int]]: + """ + Extract treedef_id from rank_name if it exists in the format 'rank_name~>treedef_id'. + """ + match = re.match(r'(.*)~>(\d+)$', rank_name) + if match: + rank_name, treedef_id_str = match.groups() + return rank_name, int(treedef_id_str) + return rank_name, None + + rank_name, extracted_treedef_id = extract_treedef_id(rank_name) + target_treedef_id = get_treedef_id(rank_name, tree, extracted_treedef_id or treedef_id, base_treedef_id) - # Return an instance of TreeRank - return TreeRank(rank_name=rank_name, treedef_id=final_treedef_id, tree=tree) + return TreeRank(rank_name, target_treedef_id, tree.lower()) def check_rank(self) -> bool: """ @@ -300,7 +312,10 @@ def process_row_item(row_key: str, row_value: Any) -> List[TreeRankCell]: ] def _filter_target_rank_columns(self, ranks_columns_in_row_not_null, target_rank_treedef_id) -> List[TreeRankCell]: - # Filter ranks_columns_in_row_not_null to only include columns that are part of the target treedef + """ + Filter ranks_columns_in_row_not_null to only include columns that are part of the target treedef + """ + return list( filter( lambda rank_column: rank_column.treedef_id == target_rank_treedef_id, @@ -332,7 +347,7 @@ def handle_multiple_or_no_treedefs(targeted_treedefids: Set[int], ranks_columns: return self, None elif len(targeted_treedefids) > 1: logger.warning(f"Multiple treedefs found in row: {targeted_treedefids}") - return self, WorkBenchParseFailure('multipleRanksInRow', {}, ranks_columns[0]) + return self, WorkBenchParseFailure('multipleRanksInRow', {}, ranks_columns[0].treedefitem_name) return None # Retrieve the target rank treedef diff --git a/specifyweb/workbench/upload/upload_plan_schema.py b/specifyweb/workbench/upload/upload_plan_schema.py index 997d0855c2a..0b0f89e4e25 100644 --- a/specifyweb/workbench/upload/upload_plan_schema.py +++ b/specifyweb/workbench/upload/upload_plan_schema.py @@ -312,7 +312,6 @@ def defer_scope_upload_table(default_collection, table: Table, to_parse: Dict, d def parse_tree_record(collection, table: Table, to_parse: Dict, base_treedefid: Optional[int] = None) -> TreeRecord: """Parse tree record from the given data""" - # Parse rank details def parse_rank(name_or_cols): if isinstance(name_or_cols, str): return name_or_cols, {'name': parse_column_options(name_or_cols)}, None @@ -321,7 +320,6 @@ def parse_rank(name_or_cols): treedefid = name_or_cols.get('treeId') return rank_name, tree_node_cols, treedefid - # Create tree rank record def create_tree_rank_record(rank, table_name, treedefid, base_treedefid): return TreeRank.create(rank, table_name, treedefid, base_treedefid).tree_rank_record() @@ -339,7 +337,7 @@ def aggregate_ranks(acc, item): acc[1] = treedefid return acc - ranks, base_treedefid = reduce(aggregate_ranks, to_parse['ranks'].items(), ({}, base_treedefid)) + ranks, base_treedefid = reduce(aggregate_ranks, to_parse['ranks'].items(), [{}, base_treedefid]) return ranks, base_treedefid # Validate ranks by checking if 'name' is present in each rank From d2f7531521ba8633896349de3908063b0dd05f60 Mon Sep 17 00:00:00 2001 From: alec_dev Date: Mon, 9 Sep 2024 12:04:35 -0500 Subject: [PATCH 67/78] edit _filter_target_rank_columns and extract_treedef_id --- specifyweb/workbench/upload/treerecord.py | 24 +++++------------------ 1 file changed, 5 insertions(+), 19 deletions(-) diff --git a/specifyweb/workbench/upload/treerecord.py b/specifyweb/workbench/upload/treerecord.py index 928644e155e..c72fba42e10 100644 --- a/specifyweb/workbench/upload/treerecord.py +++ b/specifyweb/workbench/upload/treerecord.py @@ -140,10 +140,11 @@ def extract_treedef_id(rank_name: str) -> Tuple[Union[str, Any], Optional[int]]: """ Extract treedef_id from rank_name if it exists in the format 'rank_name~>treedef_id'. """ - match = re.match(r'(.*)~>(\d+)$', rank_name) - if match: - rank_name, treedef_id_str = match.groups() - return rank_name, int(treedef_id_str) + parts = rank_name.rsplit('~>', 1) + if len(parts) == 2 and parts[1].isdigit(): + rank_name = parts[0] + tree_def_id = int(parts[1]) + return rank_name, tree_def_id return rank_name, None rank_name, extracted_treedef_id = extract_treedef_id(rank_name) @@ -311,18 +312,6 @@ def process_row_item(row_key: str, row_value: Any) -> List[TreeRankCell]: for cell in process_row_item(row_key, row_value) ] - def _filter_target_rank_columns(self, ranks_columns_in_row_not_null, target_rank_treedef_id) -> List[TreeRankCell]: - """ - Filter ranks_columns_in_row_not_null to only include columns that are part of the target treedef - """ - - return list( - filter( - lambda rank_column: rank_column.treedef_id == target_rank_treedef_id, - ranks_columns_in_row_not_null, - ) - ) - def rescope_tree_from_row(self, row: Row) -> Tuple["ScopedTreeRecord", Optional["WorkBenchParseFailure"]]: """Rescope tree from row data.""" @@ -404,9 +393,6 @@ def check_root(root): target_rank_treedef_id = targeted_treedefids.pop() target_rank_treedef = get_target_rank_treedef(tree_def_model, target_rank_treedef_id) - # Filter ranks_columns_in_row_not_null to only include columns that are part of the target treedef - # ranks_columns = self._filter_target_rank_columns(ranks_columns_in_row_not_null, target_rank_treedef_id) - # Based on the target treedef, get the treedefitems and root for the tree treedefitems = list(tree_rank_model.objects.filter(treedef_id=target_rank_treedef_id).order_by("rankid")) root = tree_node_model.objects.filter(definition_id=target_rank_treedef_id, parent=None).first() From e38d54c22ef2f251858294803f60f99ea995bcd5 Mon Sep 17 00:00:00 2001 From: alec_dev Date: Mon, 9 Sep 2024 16:59:46 -0500 Subject: [PATCH 68/78] remove base_treedef_id --- specifyweb/workbench/upload/scoping.py | 4 +- specifyweb/workbench/upload/treerecord.py | 31 +++++----- .../workbench/upload/upload_plan_schema.py | 57 +++++++------------ 3 files changed, 36 insertions(+), 56 deletions(-) diff --git a/specifyweb/workbench/upload/scoping.py b/specifyweb/workbench/upload/scoping.py index 9e1f9338491..ab70b7b3c83 100644 --- a/specifyweb/workbench/upload/scoping.py +++ b/specifyweb/workbench/upload/scoping.py @@ -188,8 +188,6 @@ def apply_scoping_to_treerecord(tr: TreeRecord, collection) -> ScopedTreeRecord: treedef = None if table.name == 'Taxon': - if tr.base_treedef_id is not None: - treedef = models.Taxontreedef.objects.filter(id=tr.base_treedef_id).first() if treedef is None: treedef = collection.discipline.taxontreedef @@ -222,7 +220,7 @@ def apply_scoping_to_treerecord(tr: TreeRecord, collection) -> ScopedTreeRecord: scoped_ranks: Dict[TreeRankRecord, Dict[str, ExtendedColumnOptions]] = {} for r, cols in tr.ranks.items(): if isinstance(r, str): - r = TreeRank.create(r, table.name, treedef.id if treedef else None, tr.base_treedef_id).tree_rank_record() + r = TreeRank.create(r, table.name, treedef.id if treedef else None).tree_rank_record() scoped_ranks[r] = {} for f, colopts in cols.items(): scoped_ranks[r][f] = extend_columnoptions(colopts, collection, table.name, f) diff --git a/specifyweb/workbench/upload/treerecord.py b/specifyweb/workbench/upload/treerecord.py index c72fba42e10..5595589a89c 100644 --- a/specifyweb/workbench/upload/treerecord.py +++ b/specifyweb/workbench/upload/treerecord.py @@ -38,7 +38,6 @@ def create( rank_name: str, tree: str, treedef_id: Optional[int] = None, - base_treedef_id: Optional[int] = None, ) -> 'TreeRank': """ Create a TreeRank instance with the given rank name, tree, and optional treedef IDs. @@ -64,34 +63,34 @@ def add_treedef_id_if_present(kwargs: Dict[str, Any], treedef_id: Optional[int]) filter_kwargs = add_treedef_id_if_present(filter_kwargs, treedef_id) return filter_kwargs - def filter_by_base_treedef_id(treedefitems, rank_name: str, base_treedef_id: Optional[int]): + def filter_by_treedef_id(treedefitems, rank_name: str): """ - Filter treedefitems by base_treedef_id and ensure only one item is found. + Filter treedefitems by treedef_id and ensure only one item is found. """ - # Check if base treedef ID is present - def check_base_treedef_id(base_treedef_id): - if base_treedef_id is None: + # Check if treedef ID is present + def check_treedef_id(treedef_id): + if treedef_id is None: raise ValueError(f"Multiple treedefitems found for rank {rank_name}") - # Filter treedefitems by base treedef ID - def filter_items_by_treedef_id(treedefitems, base_treedef_id): - return treedefitems.filter(treedef_id=base_treedef_id) + # Filter treedefitems by treedef ID + def filter_items_by_treedef_id(treedefitems, treedef_id): + return treedefitems.filter(treedef_id=treedef_id) # Validate filtered items def validate_filtered_items(treedefitems): if not treedefitems.exists(): raise ValueError( - f"No treedefitems found for rank {rank_name} and base treedef {base_treedef_id}" + f"No treedefitems found for rank {rank_name} and treedef {treedef_id}" ) if treedefitems.count() > 1: raise ValueError( - f"Multiple treedefitems found for rank {rank_name} and base treedef {base_treedef_id}" + f"Multiple treedefitems found for rank {rank_name} and treedef {treedef_id}" ) # Check base treedef ID and filter treedefitems, then validate - check_base_treedef_id(base_treedef_id) - filtered_items = filter_items_by_treedef_id(treedefitems, base_treedef_id) + check_treedef_id(treedef_id) + filtered_items = filter_items_by_treedef_id(treedefitems, treedef_id) validate_filtered_items(filtered_items) return filtered_items @@ -99,7 +98,6 @@ def get_treedef_id( rank_name: str, tree: str, treedef_id: Optional[int], - base_treedef_id: Optional[int], ) -> int: """ Get the treedef ID for the given rank name and tree. @@ -119,7 +117,7 @@ def get_first_item(treedefitems): # Handle cases where multiple items are found def handle_multiple_items(treedefitems): if treedefitems.count() > 1: - return filter_by_base_treedef_id(treedefitems, rank_name, base_treedef_id) + return filter_by_treedef_id(treedefitems, rank_name, treedef_id) return treedefitems # Build filter keyword arguments and fetch treedefitems @@ -148,7 +146,7 @@ def extract_treedef_id(rank_name: str) -> Tuple[Union[str, Any], Optional[int]]: return rank_name, None rank_name, extracted_treedef_id = extract_treedef_id(rank_name) - target_treedef_id = get_treedef_id(rank_name, tree, extracted_treedef_id or treedef_id, base_treedef_id) + target_treedef_id = get_treedef_id(rank_name, tree, extracted_treedef_id or treedef_id) return TreeRank(rank_name, target_treedef_id, tree.lower()) @@ -223,7 +221,6 @@ def validate_rank(self, tree) -> None: class TreeRecord(NamedTuple): name: str ranks: Dict[Union[str, TreeRankRecord], Dict[str, ColumnOptions]] - base_treedef_id: Optional[int] = None def apply_scoping(self, collection) -> "ScopedTreeRecord": from .scoping import apply_scoping_to_treerecord as apply_scoping diff --git a/specifyweb/workbench/upload/upload_plan_schema.py b/specifyweb/workbench/upload/upload_plan_schema.py index 0b0f89e4e25..e148307fa46 100644 --- a/specifyweb/workbench/upload/upload_plan_schema.py +++ b/specifyweb/workbench/upload/upload_plan_schema.py @@ -35,12 +35,6 @@ 'additionalProperties': False }, ] - }, - "treeId": { - "oneOf": [ - { "type": "integer" }, - { "type": "null" } - ] } }, 'required': [ 'baseTableName', 'uploadable' ], @@ -233,17 +227,14 @@ def parse_plan_with_basetable(collection, to_parse: Dict) -> Tuple[Table, Uploadable]: base_table = datamodel.get_table_strict(to_parse['baseTableName']) - extra = {} - if 'treeId' in to_parse.keys(): - extra['treedefid'] = to_parse['treeId'] - return base_table, parse_uploadable(collection, base_table, to_parse['uploadable'], extra) + return base_table, parse_uploadable(collection, base_table, to_parse['uploadable']) def parse_plan(collection, to_parse: Dict) -> Uploadable: return parse_plan_with_basetable(collection, to_parse)[1] -def parse_uploadable(collection, table: Table, to_parse: Dict, extra: Dict = {}) -> Uploadable: +def parse_uploadable(collection, table: Table, to_parse: Dict) -> Uploadable: if 'uploadTable' in to_parse: - return parse_upload_table(collection, table, to_parse['uploadTable'], extra) + return parse_upload_table(collection, table, to_parse['uploadTable']) if 'oneToOneTable' in to_parse: return OneToOneTable(*parse_upload_table(collection, table, to_parse['oneToOneTable'])) @@ -255,14 +246,11 @@ def parse_uploadable(collection, table: Table, to_parse: Dict, extra: Dict = {}) return MustMatchTreeRecord(*parse_tree_record(collection, table, to_parse['mustMatchTreeRecord'])) if 'treeRecord' in to_parse: - treedefid = None - if 'treedefid' in extra.keys(): - treedefid = int(extra['treedefid']) - return parse_tree_record(collection, table, to_parse['treeRecord'], treedefid) + return parse_tree_record(collection, table, to_parse['treeRecord']) raise ValueError('unknown uploadable type') -def parse_upload_table(collection, table: Table, to_parse: Dict, extra: Dict = {}) -> UploadTable: +def parse_upload_table(collection, table: Table, to_parse: Dict) -> UploadTable: def rel_table(key: str) -> Table: return datamodel.get_table_strict(table.get_relationship(key).relatedModelName) @@ -304,12 +292,12 @@ def defer_scope_upload_table(default_collection, table: Table, to_parse: Dict, d for key, to_one in to_parse['toOne'].items() }, toMany={ - key: [parse_to_many_record(collection, rel_table(key), record, extra) for record in to_manys] + key: [parse_to_many_record(collection, rel_table(key), record) for record in to_manys] for key, to_manys in to_parse['toMany'].items() } ) -def parse_tree_record(collection, table: Table, to_parse: Dict, base_treedefid: Optional[int] = None) -> TreeRecord: +def parse_tree_record(collection, table: Table, to_parse: Dict) -> TreeRecord: """Parse tree record from the given data""" def parse_rank(name_or_cols): @@ -320,41 +308,38 @@ def parse_rank(name_or_cols): treedefid = name_or_cols.get('treeId') return rank_name, tree_node_cols, treedefid - def create_tree_rank_record(rank, table_name, treedefid, base_treedefid): - return TreeRank.create(rank, table_name, treedefid, base_treedefid).tree_rank_record() + def create_tree_rank_record(rank, table_name, treedefid): + return TreeRank.create(rank, table_name, treedefid).tree_rank_record() - def parse_ranks(to_parse, table_name, base_treedefid): - def parse_single_rank(rank, name_or_cols, base_treedefid): + def parse_ranks(to_parse, table_name): + def parse_single_rank(rank, name_or_cols): rank_name, parsed_cols, treedefid = parse_rank(name_or_cols) - tree_rank_record = create_tree_rank_record(rank, table_name, treedefid, base_treedefid) - return tree_rank_record, parsed_cols, tree_rank_record.treedef_id + tree_rank_record = create_tree_rank_record(rank, table_name, treedefid) + return tree_rank_record, parsed_cols def aggregate_ranks(acc, item): rank, name_or_cols = item - tree_rank_record, parsed_cols, treedefid = parse_single_rank(rank, name_or_cols, acc[1]) - acc[0][tree_rank_record] = parsed_cols - if acc[1] is None: - acc[1] = treedefid + tree_rank_record, parsed_cols = parse_single_rank(rank, name_or_cols) + acc[tree_rank_record] = parsed_cols return acc - ranks, base_treedefid = reduce(aggregate_ranks, to_parse['ranks'].items(), [{}, base_treedefid]) - return ranks, base_treedefid + ranks = reduce(aggregate_ranks, to_parse['ranks'].items(), {}) + return ranks # Validate ranks by checking if 'name' is present in each rank def validate_ranks(ranks, to_parse): for rank, cols in ranks.items(): assert 'name' in cols, to_parse - ranks, base_treedefid = parse_ranks(to_parse, table.name, base_treedefid) + ranks = parse_ranks(to_parse, table.name) validate_ranks(ranks, to_parse) return TreeRecord( name=table.django_name, - ranks=ranks, - base_treedef_id=base_treedefid + ranks=ranks ) -def parse_to_many_record(collection, table: Table, to_parse: Dict, extra: Dict = {}) -> ToManyRecord: +def parse_to_many_record(collection, table: Table, to_parse: Dict) -> ToManyRecord: def rel_table(key: str) -> Table: return datamodel.get_table_strict(table.get_relationship(key).relatedModelName) @@ -364,7 +349,7 @@ def rel_table(key: str) -> Table: wbcols={k: parse_column_options(v) for k,v in to_parse['wbcols'].items()}, static=to_parse['static'], toOne={ - key: parse_uploadable(collection, rel_table(key), to_one, extra) + key: parse_uploadable(collection, rel_table(key), to_one) for key, to_one in to_parse['toOne'].items() }, ) From f3e052a66eb1a3d12928ebda4978b69e6d854208 Mon Sep 17 00:00:00 2001 From: alec_dev Date: Tue, 10 Sep 2024 09:35:47 -0500 Subject: [PATCH 69/78] scoped_ranks rewrite --- specifyweb/workbench/upload/scoping.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/specifyweb/workbench/upload/scoping.py b/specifyweb/workbench/upload/scoping.py index ab70b7b3c83..c75d90d4b16 100644 --- a/specifyweb/workbench/upload/scoping.py +++ b/specifyweb/workbench/upload/scoping.py @@ -217,13 +217,19 @@ def apply_scoping_to_treerecord(tr: TreeRecord, collection) -> ScopedTreeRecord: root = list(getattr(models, table.name.capitalize()).objects.filter(definitionitem=treedefitems[0])[:1]) # assume there is only one - scoped_ranks: Dict[TreeRankRecord, Dict[str, ExtendedColumnOptions]] = {} - for r, cols in tr.ranks.items(): - if isinstance(r, str): - r = TreeRank.create(r, table.name, treedef.id if treedef else None).tree_rank_record() - scoped_ranks[r] = {} - for f, colopts in cols.items(): - scoped_ranks[r][f] = extend_columnoptions(colopts, collection, table.name, f) + scoped_ranks: Dict[TreeRankRecord, Dict[str, ExtendedColumnOptions]] = { + ( + TreeRank.create( + r, table.name, treedef.id if treedef else None + ).tree_rank_record() + if isinstance(r, str) + else r + ): { + f: extend_columnoptions(colopts, collection, table.name, f) + for f, colopts in cols.items() + } + for r, cols in tr.ranks.items() + } return ScopedTreeRecord( name=tr.name, From 7c313d30af613ce82e4a0ca5cfebea412b2a4aa3 Mon Sep 17 00:00:00 2001 From: alec_dev Date: Tue, 10 Sep 2024 10:01:57 -0500 Subject: [PATCH 70/78] misc --- specifyweb/workbench/upload/treerecord.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specifyweb/workbench/upload/treerecord.py b/specifyweb/workbench/upload/treerecord.py index 5595589a89c..6c57a182518 100644 --- a/specifyweb/workbench/upload/treerecord.py +++ b/specifyweb/workbench/upload/treerecord.py @@ -379,8 +379,8 @@ def check_root(root): return result # Determine the target treedef based on the columns that are not null - targeted_treedefids = set([rank_column.treedef_id for rank_column in ranks_columns_in_row_not_null]) - if targeted_treedefids is None or len(targeted_treedefids) == 0: + targeted_treedefids = {rank_column.treedef_id for rank_column in ranks_columns_in_row_not_null} + if not targeted_treedefids: # return self, WorkBenchParseFailure('noRanksInRow', {}, None) return self, None elif len(targeted_treedefids) > 1: From e56e82c7fc17d362f986ff7f7751b052d908305f Mon Sep 17 00:00:00 2001 From: Sharad S Date: Tue, 10 Sep 2024 12:11:57 -0400 Subject: [PATCH 71/78] Move rank key functions to uploadPlanBuilder and uploadPlanParser --- .../components/WbPlanView/mappingHelpers.ts | 22 ------------------- .../WbPlanView/uploadPlanBuilder.ts | 15 ++++++++++++- .../components/WbPlanView/uploadPlanParser.ts | 11 +++++++++- 3 files changed, 24 insertions(+), 24 deletions(-) diff --git a/specifyweb/frontend/js_src/lib/components/WbPlanView/mappingHelpers.ts b/specifyweb/frontend/js_src/lib/components/WbPlanView/mappingHelpers.ts index 401a9623288..dc65961182d 100644 --- a/specifyweb/frontend/js_src/lib/components/WbPlanView/mappingHelpers.ts +++ b/specifyweb/frontend/js_src/lib/components/WbPlanView/mappingHelpers.ts @@ -111,28 +111,6 @@ export const formatTreeDefinition = (definitionName: string): string => export const formatTreeRank = (rankName: string): string => `${schema.treeRankSymbol}${rankName}`; -// Delimiter used for rank name keys i.e: ~> -const RANK_KEY_DELIMITER = '~>'; - -/** - * Returns a formatted tree rank name along with its tree id: (e.x $Kingdom => $Kingdom~>1) - * Used for generating unique key names when there are multiple trees with the same rank name. - * Opposite of getRankNameWithoutTreeId - * See: https://github.com/specify/specify7/pull/5091#issuecomment-2328037741 - */ -export const formatTreeRankWithTreeId = ( - rankName: string, - treeId: number -): string => `${rankName}${RANK_KEY_DELIMITER}${treeId}`; - -/** - * Returns the tree rank name after stripping its tree id: (e.x Kingdom~>1 => Kingdom) - * NOTE: Does not consider whether rankName is formatted with $ or not. - * Opposite of formatTreeRankWithTreeId - */ -export const getRankNameWithoutTreeId = (rankName: string): string => - rankName.split(RANK_KEY_DELIMITER)[0]; - // Match fields names like startDate_fullDate, but not _formatted export const valueIsPartialField = (value: string): boolean => value.includes(schema.fieldPartSeparator) && diff --git a/specifyweb/frontend/js_src/lib/components/WbPlanView/uploadPlanBuilder.ts b/specifyweb/frontend/js_src/lib/components/WbPlanView/uploadPlanBuilder.ts index 0fbc2d45f3b..83c61811ba0 100644 --- a/specifyweb/frontend/js_src/lib/components/WbPlanView/uploadPlanBuilder.ts +++ b/specifyweb/frontend/js_src/lib/components/WbPlanView/uploadPlanBuilder.ts @@ -6,7 +6,6 @@ import type { Tables } from '../DataModel/types'; import { getTreeDefinitions, isTreeTable } from '../InitialContext/treeRanks'; import { defaultColumnOptions } from './linesGetter'; import type { SplitMappingPath } from './mappingHelpers'; -import { formatTreeRankWithTreeId } from './mappingHelpers'; import { getNameFromTreeDefinitionName, valueIsTreeDefinition, @@ -212,3 +211,17 @@ const indexMappings = ( ] as const ) ); + +// Delimiter used for rank name keys i.e: ~> +export const RANK_KEY_DELIMITER = '~>'; + +/** + * Returns a formatted tree rank name along with its tree id: (e.x Kingdom => Kingdom~>1) + * Used for generating unique key names in the upload plan when there are multiple trees with the same rank name. + * Opposite of uploadPlanParser.ts > getRankNameWithoutTreeId() + * See: https://github.com/specify/specify7/pull/5091#issuecomment-2328037741 + */ +const formatTreeRankWithTreeId = ( + rankName: string, + treeId: number +): string => `${rankName}${RANK_KEY_DELIMITER}${treeId}`; diff --git a/specifyweb/frontend/js_src/lib/components/WbPlanView/uploadPlanParser.ts b/specifyweb/frontend/js_src/lib/components/WbPlanView/uploadPlanParser.ts index e3e54e73127..718fb137b39 100644 --- a/specifyweb/frontend/js_src/lib/components/WbPlanView/uploadPlanParser.ts +++ b/specifyweb/frontend/js_src/lib/components/WbPlanView/uploadPlanParser.ts @@ -6,9 +6,10 @@ import { softFail } from '../Errors/Crash'; import { getTreeDefinitions } from '../InitialContext/treeRanks'; import { defaultColumnOptions } from './linesGetter'; import type { MappingPath } from './Mapper'; -import { getRankNameWithoutTreeId, SplitMappingPath } from './mappingHelpers'; +import { SplitMappingPath } from './mappingHelpers'; import { formatTreeDefinition } from './mappingHelpers'; import { formatToManyIndex, formatTreeRank } from './mappingHelpers'; +import { RANK_KEY_DELIMITER } from './uploadPlanBuilder'; export type MatchBehaviors = 'ignoreAlways' | 'ignoreNever' | 'ignoreWhenBlank'; @@ -212,3 +213,11 @@ export function parseUploadPlan(uploadPlan: UploadPlan): { ), }; } + +/** + * Returns the tree rank name after stripping its tree id: (e.x Kingdom~>1 => Kingdom) + * NOTE: Does not consider whether rankName is formatted with $ or not. + * Opposite of uploadPlanBuilder.ts > formatTreeRankWithTreeId() + */ +const getRankNameWithoutTreeId = (rankName: string): string => + rankName.split(RANK_KEY_DELIMITER)[0]; From 10324c97095b34c296b37d3a18a354dba7a95e6c Mon Sep 17 00:00:00 2001 From: Sharad S Date: Tue, 10 Sep 2024 16:15:25 +0000 Subject: [PATCH 72/78] Lint code with ESLint and Prettier Triggered by e56e82c7fc17d362f986ff7f7751b052d908305f on branch refs/heads/issue-4980 --- .../js_src/lib/components/WbPlanView/uploadPlanBuilder.ts | 6 ++---- .../js_src/lib/components/WbPlanView/uploadPlanParser.ts | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/specifyweb/frontend/js_src/lib/components/WbPlanView/uploadPlanBuilder.ts b/specifyweb/frontend/js_src/lib/components/WbPlanView/uploadPlanBuilder.ts index 83c61811ba0..1ffd74284af 100644 --- a/specifyweb/frontend/js_src/lib/components/WbPlanView/uploadPlanBuilder.ts +++ b/specifyweb/frontend/js_src/lib/components/WbPlanView/uploadPlanBuilder.ts @@ -221,7 +221,5 @@ export const RANK_KEY_DELIMITER = '~>'; * Opposite of uploadPlanParser.ts > getRankNameWithoutTreeId() * See: https://github.com/specify/specify7/pull/5091#issuecomment-2328037741 */ -const formatTreeRankWithTreeId = ( - rankName: string, - treeId: number -): string => `${rankName}${RANK_KEY_DELIMITER}${treeId}`; +const formatTreeRankWithTreeId = (rankName: string, treeId: number): string => + `${rankName}${RANK_KEY_DELIMITER}${treeId}`; diff --git a/specifyweb/frontend/js_src/lib/components/WbPlanView/uploadPlanParser.ts b/specifyweb/frontend/js_src/lib/components/WbPlanView/uploadPlanParser.ts index 718fb137b39..f5220b43d2e 100644 --- a/specifyweb/frontend/js_src/lib/components/WbPlanView/uploadPlanParser.ts +++ b/specifyweb/frontend/js_src/lib/components/WbPlanView/uploadPlanParser.ts @@ -6,7 +6,7 @@ import { softFail } from '../Errors/Crash'; import { getTreeDefinitions } from '../InitialContext/treeRanks'; import { defaultColumnOptions } from './linesGetter'; import type { MappingPath } from './Mapper'; -import { SplitMappingPath } from './mappingHelpers'; +import type { SplitMappingPath } from './mappingHelpers'; import { formatTreeDefinition } from './mappingHelpers'; import { formatToManyIndex, formatTreeRank } from './mappingHelpers'; import { RANK_KEY_DELIMITER } from './uploadPlanBuilder'; From e8a9ed4b00fbdf883ac7192b5cfd0eef652c8e85 Mon Sep 17 00:00:00 2001 From: Caroline D <108160931+CarolineDenis@users.noreply.github.com> Date: Wed, 11 Sep 2024 07:42:20 -0700 Subject: [PATCH 73/78] Add tree name in front of rank --- .../components/WbPlanView/uploadPlanBuilder.ts | 15 ++++----------- .../js_src/lib/tests/fixtures/uploadplan.1.json | 12 ++++++------ 2 files changed, 10 insertions(+), 17 deletions(-) diff --git a/specifyweb/frontend/js_src/lib/components/WbPlanView/uploadPlanBuilder.ts b/specifyweb/frontend/js_src/lib/components/WbPlanView/uploadPlanBuilder.ts index 1ffd74284af..3082316a7e4 100644 --- a/specifyweb/frontend/js_src/lib/components/WbPlanView/uploadPlanBuilder.ts +++ b/specifyweb/frontend/js_src/lib/components/WbPlanView/uploadPlanBuilder.ts @@ -63,7 +63,7 @@ const toTreeRecordVariety = (lines: RA): TreeRecord => { ? [[fullName, rankMappedFields]] : indexMappings(rankMappedFields); - const treeName = valueIsTreeDefinition(fullName) + const treeName: string | undefined = valueIsTreeDefinition(fullName) ? getNameFromTreeDefinitionName(fullName) : undefined; @@ -83,7 +83,9 @@ const toTreeRecordVariety = (lines: RA): TreeRecord => { return [ treeId === undefined ? rankName - : formatTreeRankWithTreeId(rankName, treeId), + : `${ + treeName === undefined ? '' : treeName + RANK_KEY_DELIMITER + }${rankName}`, { treeNodeCols: toTreeRecordRanks(mappedFields), treeId, @@ -214,12 +216,3 @@ const indexMappings = ( // Delimiter used for rank name keys i.e: ~> export const RANK_KEY_DELIMITER = '~>'; - -/** - * Returns a formatted tree rank name along with its tree id: (e.x Kingdom => Kingdom~>1) - * Used for generating unique key names in the upload plan when there are multiple trees with the same rank name. - * Opposite of uploadPlanParser.ts > getRankNameWithoutTreeId() - * See: https://github.com/specify/specify7/pull/5091#issuecomment-2328037741 - */ -const formatTreeRankWithTreeId = (rankName: string, treeId: number): string => - `${rankName}${RANK_KEY_DELIMITER}${treeId}`; diff --git a/specifyweb/frontend/js_src/lib/tests/fixtures/uploadplan.1.json b/specifyweb/frontend/js_src/lib/tests/fixtures/uploadplan.1.json index 26476e857f6..a639bc1327d 100644 --- a/specifyweb/frontend/js_src/lib/tests/fixtures/uploadplan.1.json +++ b/specifyweb/frontend/js_src/lib/tests/fixtures/uploadplan.1.json @@ -275,7 +275,7 @@ "taxon": { "mustMatchTreeRecord": { "ranks": { - "Class~>1": { + "Class": { "treeNodeCols": { "name": { "matchBehavior": "ignoreAlways", @@ -286,32 +286,32 @@ }, "treeId": 1 }, - "Family~>1": { + "Family": { "treeNodeCols": { "name": "Family" }, "treeId": 1 }, - "Genus~>1": { + "Genus": { "treeNodeCols": { "name": "Genus" }, "treeId": 1 }, - "Subgenus~>1": { + "Subgenus": { "treeNodeCols": { "name": "Subgenus" }, "treeId": 1 }, - "Species~>1": { + "Species": { "treeNodeCols": { "name": "Species", "author": "Species Author" }, "treeId": 1 }, - "Subspecies~>1": { + "Subspecies": { "treeNodeCols": { "name": "Subspecies", "author": "Subspecies Author" From e00add22bee6e62590340eae745a777f1f851750 Mon Sep 17 00:00:00 2001 From: alec_dev Date: Wed, 11 Sep 2024 11:30:41 -0500 Subject: [PATCH 74/78] catch multipleRanksForTreedef WorkBenchParseFailure --- specifyweb/workbench/upload/treerecord.py | 44 +++++++++++++++++------ 1 file changed, 33 insertions(+), 11 deletions(-) diff --git a/specifyweb/workbench/upload/treerecord.py b/specifyweb/workbench/upload/treerecord.py index 6c57a182518..2934f5fcfff 100644 --- a/specifyweb/workbench/upload/treerecord.py +++ b/specifyweb/workbench/upload/treerecord.py @@ -2,9 +2,8 @@ For uploading tree records. """ -from collections import namedtuple +from itertools import groupby import logging -import re from typing import List, Dict, Any, Tuple, NamedTuple, Optional, Union, Set from django.db import transaction, IntegrityError @@ -265,7 +264,7 @@ def get_treedefs(self) -> Set: tree_def_model = get_treedef_model(self.name) treedefids = set([tree_rank_record.treedef_id for tree_rank_record in self.ranks.keys()]) return set(tree_def_model.objects.filter(id__in=treedefids)) - + def _get_not_null_ranks_columns_in_row(self, row: Row) -> List[TreeRankCell]: """ Get rank columns that are not null in the row. @@ -308,7 +307,7 @@ def process_row_item(row_key: str, row_value: Any) -> List[TreeRankCell]: for row_key, row_value in row.items() for cell in process_row_item(row_key, row_value) ] - + def rescope_tree_from_row(self, row: Row) -> Tuple["ScopedTreeRecord", Optional["WorkBenchParseFailure"]]: """Rescope tree from row data.""" @@ -328,12 +327,34 @@ def get_targeted_treedefids(ranks_columns: List[TreeRankCell]) -> Set[int]: return set(rank_column.treedef_id for rank_column in ranks_columns) # Handle cases where there are multiple or no treedefs - def handle_multiple_or_no_treedefs(targeted_treedefids: Set[int], ranks_columns: List[TreeRankCell]): + def handle_multiple_or_no_treedefs( + targeted_treedefids: Set[int], ranks_columns: List[TreeRankCell] + ) -> Tuple["ScopedTreeRecord", Optional["WorkBenchParseFailure"]]: if not targeted_treedefids: return self, None elif len(targeted_treedefids) > 1: logger.warning(f"Multiple treedefs found in row: {targeted_treedefids}") return self, WorkBenchParseFailure('multipleRanksInRow', {}, ranks_columns[0].treedefitem_name) + + # Group ranks_columns by treedef_id using itertools.groupby + ranks_columns.sort(key=lambda x: x.treedef_id) # groupby requires sorted input + grouped_by_treedef_id = {k: list(v) for k, v in groupby(ranks_columns, key=lambda x: x.treedef_id)} + + # Check if any treedef_id has more than one rank + multiple_ranks = any(len(columns) > 1 for columns in grouped_by_treedef_id.values()) + if multiple_ranks: + treedef_id = next( + treedef_id + for treedef_id, columns in grouped_by_treedef_id.items() + if len(columns) > 1 + ) + logger.warning(f"Multiple ranks found for treedef_id {treedef_id}") + return self, WorkBenchParseFailure( + "multipleRanksForTreedef", + {}, + grouped_by_treedef_id[treedef_id][0].treedefitem_name, + ) + return None # Retrieve the target rank treedef @@ -360,7 +381,7 @@ def check_root(root): treedefitems = fetch_treedefitems() root = fetch_root() root_checked = check_root(root) - + if root_checked is not root: return root_checked @@ -368,9 +389,6 @@ def check_root(root): tree_def_model, tree_rank_model, tree_node_model = get_models(self.name) - if len(set(tr.treedef_id for tr in self.ranks.keys())) == 1: - return self, None - ranks_columns_in_row_not_null = get_not_null_ranks_columns(row) targeted_treedefids = get_targeted_treedefids(ranks_columns_in_row_not_null) @@ -378,6 +396,10 @@ def check_root(root): if result: return result + unique_treedef_ids = {tr.treedef_id for tr in self.ranks.keys()} + if len(unique_treedef_ids) == 1: + return self, None + # Determine the target treedef based on the columns that are not null targeted_treedefids = {rank_column.treedef_id for rank_column in ranks_columns_in_row_not_null} if not targeted_treedefids: @@ -386,11 +408,11 @@ def check_root(root): elif len(targeted_treedefids) > 1: logger.warning(f"Multiple treedefs found in row: {targeted_treedefids}") return self, WorkBenchParseFailure('multipleRanksInRow', {}, list(ranks_columns_in_row_not_null)[0].treedefitem_name) - + target_rank_treedef_id = targeted_treedefids.pop() target_rank_treedef = get_target_rank_treedef(tree_def_model, target_rank_treedef_id) - # Based on the target treedef, get the treedefitems and root for the tree + # Based on the target treedef, get the treedefitems and root for the tree treedefitems = list(tree_rank_model.objects.filter(treedef_id=target_rank_treedef_id).order_by("rankid")) root = tree_node_model.objects.filter(definition_id=target_rank_treedef_id, parent=None).first() From 20bfd8af0f7308d332139e0a99d37f7718ec919d Mon Sep 17 00:00:00 2001 From: alec_dev Date: Wed, 11 Sep 2024 11:49:42 -0500 Subject: [PATCH 75/78] edit accepted formatting to treedef_id~>rank_name --- specifyweb/workbench/upload/treerecord.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/specifyweb/workbench/upload/treerecord.py b/specifyweb/workbench/upload/treerecord.py index 2934f5fcfff..38513150d0c 100644 --- a/specifyweb/workbench/upload/treerecord.py +++ b/specifyweb/workbench/upload/treerecord.py @@ -135,12 +135,12 @@ def handle_multiple_items(treedefitems): def extract_treedef_id(rank_name: str) -> Tuple[Union[str, Any], Optional[int]]: """ - Extract treedef_id from rank_name if it exists in the format 'rank_name~>treedef_id'. + Extract treedef_id from rank_name if it exists in the format 'treedef_id~>rank_name'. """ - parts = rank_name.rsplit('~>', 1) - if len(parts) == 2 and parts[1].isdigit(): - rank_name = parts[0] - tree_def_id = int(parts[1]) + parts = rank_name.split('~>', 1) + if len(parts) == 2 and parts[0].isdigit(): + tree_def_id = int(parts[0]) + rank_name = parts[1] return rank_name, tree_def_id return rank_name, None @@ -329,7 +329,7 @@ def get_targeted_treedefids(ranks_columns: List[TreeRankCell]) -> Set[int]: # Handle cases where there are multiple or no treedefs def handle_multiple_or_no_treedefs( targeted_treedefids: Set[int], ranks_columns: List[TreeRankCell] - ) -> Tuple["ScopedTreeRecord", Optional["WorkBenchParseFailure"]]: + ) -> Optional[Tuple["ScopedTreeRecord", Optional["WorkBenchParseFailure"]]]: if not targeted_treedefids: return self, None elif len(targeted_treedefids) > 1: From 590579ea8ac1dfe399606eaeedaa1cf2fdb61bf1 Mon Sep 17 00:00:00 2001 From: alec_dev Date: Wed, 11 Sep 2024 14:09:04 -0500 Subject: [PATCH 76/78] change extract_treedef_id to extract_treedef_name --- specifyweb/workbench/upload/treerecord.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/specifyweb/workbench/upload/treerecord.py b/specifyweb/workbench/upload/treerecord.py index 38513150d0c..b8c4d8c0c7a 100644 --- a/specifyweb/workbench/upload/treerecord.py +++ b/specifyweb/workbench/upload/treerecord.py @@ -96,7 +96,8 @@ def validate_filtered_items(treedefitems): def get_treedef_id( rank_name: str, tree: str, - treedef_id: Optional[int], + treedef_name: Optional[str] = None, + treedef_id: Optional[int] = None ) -> int: """ Get the treedef ID for the given rank name and tree. @@ -119,6 +120,9 @@ def handle_multiple_items(treedefitems): return filter_by_treedef_id(treedefitems, rank_name, treedef_id) return treedefitems + if treedef_id is None and treedef_name is not None: + treedef_id = get_treedef_model(tree).objects.get(name=treedef_name) + # Build filter keyword arguments and fetch treedefitems filter_kwargs = build_filter_kwargs(rank_name, treedef_id) treedefitems = fetch_treedefitems(filter_kwargs) @@ -133,19 +137,19 @@ def handle_multiple_items(treedefitems): return first_item.treedef_id - def extract_treedef_id(rank_name: str) -> Tuple[Union[str, Any], Optional[int]]: + def extract_treedef_name(rank_name: str) -> Tuple[str, Optional[str]]: """ - Extract treedef_id from rank_name if it exists in the format 'treedef_id~>rank_name'. + Extract treedef_name from rank_name if it exists in the format 'treedef_name~>rank_name'. """ parts = rank_name.split('~>', 1) - if len(parts) == 2 and parts[0].isdigit(): - tree_def_id = int(parts[0]) + if len(parts) == 2: + treedef_name = parts[0] rank_name = parts[1] - return rank_name, tree_def_id + return rank_name, treedef_name return rank_name, None - rank_name, extracted_treedef_id = extract_treedef_id(rank_name) - target_treedef_id = get_treedef_id(rank_name, tree, extracted_treedef_id or treedef_id) + rank_name, extracted_treedef_name = extract_treedef_name(rank_name) + target_treedef_id = get_treedef_id(rank_name, tree, extracted_treedef_name, treedef_id) return TreeRank(rank_name, target_treedef_id, tree.lower()) From c3f4862cdf8c8b9ca3e9741bf7f482a818e5ae62 Mon Sep 17 00:00:00 2001 From: alec_dev Date: Wed, 11 Sep 2024 14:13:22 -0500 Subject: [PATCH 77/78] avoid unit test failures, allow multiple ranks when only one treedef is given --- specifyweb/workbench/upload/treerecord.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/specifyweb/workbench/upload/treerecord.py b/specifyweb/workbench/upload/treerecord.py index b8c4d8c0c7a..9beee830b31 100644 --- a/specifyweb/workbench/upload/treerecord.py +++ b/specifyweb/workbench/upload/treerecord.py @@ -393,6 +393,10 @@ def check_root(root): tree_def_model, tree_rank_model, tree_node_model = get_models(self.name) + unique_treedef_ids = {tr.treedef_id for tr in self.ranks.keys()} + if len(unique_treedef_ids) == 1: + return self, None + ranks_columns_in_row_not_null = get_not_null_ranks_columns(row) targeted_treedefids = get_targeted_treedefids(ranks_columns_in_row_not_null) @@ -400,10 +404,6 @@ def check_root(root): if result: return result - unique_treedef_ids = {tr.treedef_id for tr in self.ranks.keys()} - if len(unique_treedef_ids) == 1: - return self, None - # Determine the target treedef based on the columns that are not null targeted_treedefids = {rank_column.treedef_id for rank_column in ranks_columns_in_row_not_null} if not targeted_treedefids: From eb3c8200bad92af9f43c063655ba47f8a02badf2 Mon Sep 17 00:00:00 2001 From: alec_dev Date: Wed, 11 Sep 2024 15:05:52 -0500 Subject: [PATCH 78/78] fix WorkBenchParseFailure column name --- specifyweb/workbench/upload/treerecord.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/specifyweb/workbench/upload/treerecord.py b/specifyweb/workbench/upload/treerecord.py index 9beee830b31..92e29df7729 100644 --- a/specifyweb/workbench/upload/treerecord.py +++ b/specifyweb/workbench/upload/treerecord.py @@ -4,6 +4,8 @@ from itertools import groupby import logging +from math import e +from os import error from typing import List, Dict, Any, Tuple, NamedTuple, Optional, Union, Set from django.db import transaction, IntegrityError @@ -26,6 +28,7 @@ class TreeRankCell(NamedTuple): treedefitem_name: str tree_node_attribute: str upload_value: str + column_fullname: str = "" class TreeRank(NamedTuple): rank_name: str @@ -287,12 +290,13 @@ def match_column(row_key: str, formatted_column: str) -> bool: return formatted_column == row_key # Create a TreeRankCell instance - def create_tree_rank_cell(tree_rank_record: Any, col_name: str, row_value: Any) -> TreeRankCell: + def create_tree_rank_cell(tree_rank_record: Any, col_name: str, row_value: Any, row_key: Any) -> TreeRankCell: return TreeRankCell( tree_rank_record.treedef_id, # treedefid tree_rank_record.rank_name, # treedefitem_name col_name, # tree_node_attribute row_value, # upload_value + row_key # column_fullname ) # Process the row item @@ -300,7 +304,7 @@ def process_row_item(row_key: str, row_value: Any) -> List[TreeRankCell]: if not is_not_null(row_value): return [] return [ - create_tree_rank_cell(tree_rank_record, col_name, row_value) + create_tree_rank_cell(tree_rank_record, col_name, row_value, row_key) for tree_rank_record, cols in self.ranks.items() for col_name, col_opts in cols.items() if match_column(row_key, format_column(col_name, col_opts)) @@ -338,7 +342,8 @@ def handle_multiple_or_no_treedefs( return self, None elif len(targeted_treedefids) > 1: logger.warning(f"Multiple treedefs found in row: {targeted_treedefids}") - return self, WorkBenchParseFailure('multipleRanksInRow', {}, ranks_columns[0].treedefitem_name) + error_col_name = ranks_columns[0].column_fullname + return self, WorkBenchParseFailure('multipleRanksInRow', {}, error_col_name) # Group ranks_columns by treedef_id using itertools.groupby ranks_columns.sort(key=lambda x: x.treedef_id) # groupby requires sorted input @@ -353,11 +358,8 @@ def handle_multiple_or_no_treedefs( if len(columns) > 1 ) logger.warning(f"Multiple ranks found for treedef_id {treedef_id}") - return self, WorkBenchParseFailure( - "multipleRanksForTreedef", - {}, - grouped_by_treedef_id[treedef_id][0].treedefitem_name, - ) + error_col_name = grouped_by_treedef_id[treedef_id][0].column_fullname + return self, WorkBenchParseFailure("multipleRanksForTreedef", {}, error_col_name) return None @@ -411,7 +413,8 @@ def check_root(root): return self, None elif len(targeted_treedefids) > 1: logger.warning(f"Multiple treedefs found in row: {targeted_treedefids}") - return self, WorkBenchParseFailure('multipleRanksInRow', {}, list(ranks_columns_in_row_not_null)[0].treedefitem_name) + error_col_name = list(ranks_columns_in_row_not_null)[0].column_fullname + return self, WorkBenchParseFailure("multipleRanksInRow", {}, error_col_name) target_rank_treedef_id = targeted_treedefids.pop() target_rank_treedef = get_target_rank_treedef(tree_def_model, target_rank_treedef_id)