From 9924ac850eb604f081be58efcf820502d0c6039c Mon Sep 17 00:00:00 2001 From: cosa65 Date: Wed, 29 Nov 2023 09:13:17 -0300 Subject: [PATCH 01/53] Unrelated, rename file to match its default export Signed-off-by: cosa65 --- .../[experimentId]/plots-and-tables/dot-plot/index.jsx | 4 ++-- .../[experimentId]/plots-and-tables/frequency/index.jsx | 4 ++-- src/utils/{fileNames.js => plotCsvFilename.js} | 0 3 files changed, 4 insertions(+), 4 deletions(-) rename src/utils/{fileNames.js => plotCsvFilename.js} (100%) diff --git a/src/pages/experiments/[experimentId]/plots-and-tables/dot-plot/index.jsx b/src/pages/experiments/[experimentId]/plots-and-tables/dot-plot/index.jsx index a0bf1f7973..804f994df6 100644 --- a/src/pages/experiments/[experimentId]/plots-and-tables/dot-plot/index.jsx +++ b/src/pages/experiments/[experimentId]/plots-and-tables/dot-plot/index.jsx @@ -22,7 +22,7 @@ import Header from 'components/Header'; import PlotContainer from 'components/plots/PlotContainer'; import Loader from 'components/Loader'; import ExportAsCSV from 'components/plots/ExportAsCSV'; -import fileNames from 'utils/fileNames'; +import plotCsvFilename from 'utils/plotCsvFilename'; import { updatePlotConfig, loadPlotConfig, @@ -101,7 +101,7 @@ const DotPlotPage = (props) => { const highestGenesLoadedRef = useRef(false); const experimentName = useSelector((state) => state.experimentSettings.info.experimentName); - const csvFileName = fileNames(experimentName, 'DOT_PLOT', [config?.selectedCellSet, config?.selectedPoints]); + const csvFileName = plotCsvFilename(experimentName, 'DOT_PLOT', [config?.selectedCellSet, config?.selectedPoints]); useEffect(() => { if (!config) dispatch(loadPlotConfig(experimentId, plotUuid, plotType)); diff --git a/src/pages/experiments/[experimentId]/plots-and-tables/frequency/index.jsx b/src/pages/experiments/[experimentId]/plots-and-tables/frequency/index.jsx index 57a903d9f0..563646f81f 100644 --- a/src/pages/experiments/[experimentId]/plots-and-tables/frequency/index.jsx +++ b/src/pages/experiments/[experimentId]/plots-and-tables/frequency/index.jsx @@ -25,7 +25,7 @@ import SelectCellSets from 'components/plots/styling/frequency/SelectCellSets'; import { updatePlotConfig, loadPlotConfig } from 'redux/actions/componentConfig'; import loadCellSets from 'redux/actions/cellSets/loadCellSets'; -import plotCsvFilename from 'utils/fileNames'; +import plotCsvFilename from 'utils/plotCsvFilename'; import { plotNames } from 'utils/constants'; import PlotContainer from 'components/plots/PlotContainer'; @@ -192,7 +192,7 @@ const FrequencyPlotPage = ({ experimentId }) => { return ( - { config?.legend?.showAlert && } + {config?.legend?.showAlert && }
Date: Wed, 29 Nov 2023 09:44:31 -0300 Subject: [PATCH 02/53] Replace usage of fileNames in redux with fileNamesByTech Signed-off-by: cosa65 --- .../data-management/LaunchAnalysisButton.jsx | 6 ++++-- src/utils/fileNamesByTech.js | 10 ++++++++++ 2 files changed, 14 insertions(+), 2 deletions(-) create mode 100644 src/utils/fileNamesByTech.js diff --git a/src/components/data-management/LaunchAnalysisButton.jsx b/src/components/data-management/LaunchAnalysisButton.jsx index 2ad57a81eb..3a74bc1dc0 100644 --- a/src/components/data-management/LaunchAnalysisButton.jsx +++ b/src/components/data-management/LaunchAnalysisButton.jsx @@ -13,6 +13,7 @@ import integrationTestConstants from 'utils/integrationTestConstants'; import { useAppRouter } from 'utils/AppRouteProvider'; import calculatePipelinesRerunStatus from 'utils/data-management/calculatePipelinesRerunStatus'; +import fileNamesByTech from 'utils/filenamesByTech'; const LaunchButtonTemplate = (props) => { const { @@ -100,8 +101,9 @@ const LaunchAnalysisButton = () => { const metadataKeysAvailable = activeExperiment.metadataKeys.length; const allSampleFilesUploaded = (sample) => { - // Check if all files for a given tech has been uploaded - const { fileNames } = sample; + // Check if all files for a given tech have been uploaded + const fileNames = fileNamesByTech[sample.type]; + if ( !fileUploadSpecifications[sample.type].requiredFiles.every( (file) => fileNames.includes(file.key), diff --git a/src/utils/fileNamesByTech.js b/src/utils/fileNamesByTech.js new file mode 100644 index 0000000000..5a30475494 --- /dev/null +++ b/src/utils/fileNamesByTech.js @@ -0,0 +1,10 @@ +import { sampleTech } from 'utils/constants'; + +const fileNamesByTech = { + [sampleTech['10X']]: ['features.tsv.gz', 'barcodes.tsv.gz', 'matrix.mtx.gz'], + [sampleTech.H5]: ['matrix.h5.gz'], + [sampleTech.SEURAT]: ['r.rds'], + [sampleTech.RHAPSODY]: ['expression_data.st.gz'], +}; + +export default fileNamesByTech; From ab78335cc97e8a9cca3ceed0877240d994a91ae9 Mon Sep 17 00:00:00 2001 From: cosa65 Date: Wed, 29 Nov 2023 09:49:43 -0300 Subject: [PATCH 03/53] Remove fileNames from redux Signed-off-by: cosa65 --- src/components/data-management/LaunchAnalysisButton.jsx | 2 +- src/redux/actions/samples/loadSamples.js | 8 ++------ src/redux/reducers/samples/initialState.js | 1 - src/redux/reducers/samples/samplesFileUpdate.js | 6 ------ 4 files changed, 3 insertions(+), 14 deletions(-) diff --git a/src/components/data-management/LaunchAnalysisButton.jsx b/src/components/data-management/LaunchAnalysisButton.jsx index 3a74bc1dc0..593d00f381 100644 --- a/src/components/data-management/LaunchAnalysisButton.jsx +++ b/src/components/data-management/LaunchAnalysisButton.jsx @@ -13,7 +13,7 @@ import integrationTestConstants from 'utils/integrationTestConstants'; import { useAppRouter } from 'utils/AppRouteProvider'; import calculatePipelinesRerunStatus from 'utils/data-management/calculatePipelinesRerunStatus'; -import fileNamesByTech from 'utils/filenamesByTech'; +import fileNamesByTech from 'utils/fileNamesByTech'; const LaunchButtonTemplate = (props) => { const { diff --git a/src/redux/actions/samples/loadSamples.js b/src/redux/actions/samples/loadSamples.js index 73f6fc7722..8e644ff754 100644 --- a/src/redux/actions/samples/loadSamples.js +++ b/src/redux/actions/samples/loadSamples.js @@ -12,7 +12,6 @@ const toApiV1 = (samples, experimentId) => { const apiV1Samples = {}; const buildApiv1Files = (files) => { - const fileNames = []; const apiV1Files = {}; Object.keys(files).forEach((key) => { @@ -21,8 +20,6 @@ const toApiV1 = (samples, experimentId) => { const fileName = fileNameForApiV1[fileType]; - fileNames.push(fileNameForApiV1[fileType]); - apiV1Files[fileName] = { size: files[key].size, valid: true, @@ -33,11 +30,11 @@ const toApiV1 = (samples, experimentId) => { }; }); - return { apiV1Files, fileNames }; + return apiV1Files; }; samples.forEach((sample) => { - const { apiV1Files, fileNames } = buildApiv1Files(sample.files); + const apiV1Files = buildApiv1Files(sample.files); apiV1Samples[sample.id] = { experimentId, metadata: sample.metadata, @@ -47,7 +44,6 @@ const toApiV1 = (samples, experimentId) => { files: apiV1Files, type: sample.sampleTechnology, options: sample.options, - fileNames, uuid: sample.id, }; }); diff --git a/src/redux/reducers/samples/initialState.js b/src/redux/reducers/samples/initialState.js index 3a37fa4720..3b493d309b 100644 --- a/src/redux/reducers/samples/initialState.js +++ b/src/redux/reducers/samples/initialState.js @@ -9,7 +9,6 @@ const sampleTemplate = { lastModified: null, complete: false, error: false, - fileNames: [], files: {}, metadata: {}, options: {}, diff --git a/src/redux/reducers/samples/samplesFileUpdate.js b/src/redux/reducers/samples/samplesFileUpdate.js index ee1561f15f..73bf40b805 100644 --- a/src/redux/reducers/samples/samplesFileUpdate.js +++ b/src/redux/reducers/samples/samplesFileUpdate.js @@ -18,16 +18,10 @@ const samplesFileUpdate = (state, action) => { newFile = _.merge({}, oldFile, fileDiff); } - const newFileNames = _.cloneDeep(state[sampleUuid].fileNames); - if (!newFileNames.includes(fileName)) { - newFileNames.push(fileName); - } - return { ...state, [sampleUuid]: { ...state[sampleUuid], - fileNames: newFileNames, files: { ...state[sampleUuid].files, [fileName]: { From b019072a2c5a4830964a8c766d956a5cb91e83ab Mon Sep 17 00:00:00 2001 From: cosa65 Date: Wed, 29 Nov 2023 11:11:52 -0300 Subject: [PATCH 04/53] Rename prop file to data to be more accurate in what UploadDetailsModal receives Signed-off-by: cosa65 --- .../components/data-management/UploadDetailsModal.test.jsx | 2 +- src/components/data-management/SamplesTableCells.jsx | 2 +- src/components/data-management/UploadDetailsModal.jsx | 6 +++--- .../data-management/metadata/CellLevelUploadModal.jsx | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/__test__/components/data-management/UploadDetailsModal.test.jsx b/src/__test__/components/data-management/UploadDetailsModal.test.jsx index f8cb267705..d8b5722c40 100644 --- a/src/__test__/components/data-management/UploadDetailsModal.test.jsx +++ b/src/__test__/components/data-management/UploadDetailsModal.test.jsx @@ -14,7 +14,7 @@ const mockOnRetry = jest.fn(); const defaultProps = { visible: true, - file: { + data: { name: 'example.txt', size: 1024, lastModified: new Date().toISOString(), diff --git a/src/components/data-management/SamplesTableCells.jsx b/src/components/data-management/SamplesTableCells.jsx index 9b998e10f9..240294ee1a 100644 --- a/src/components/data-management/SamplesTableCells.jsx +++ b/src/components/data-management/SamplesTableCells.jsx @@ -157,7 +157,7 @@ const UploadCell = (props) => {
{uploadDetailsModalVisible && ( setUploadDetailsModalVisible(false)} onDownload={onDownload} onDelete={() => dispatch(deleteSamples([sampleUuid]))} diff --git a/src/components/data-management/UploadDetailsModal.jsx b/src/components/data-management/UploadDetailsModal.jsx index 5e7d4ba4fa..0f5fa1991e 100644 --- a/src/components/data-management/UploadDetailsModal.jsx +++ b/src/components/data-management/UploadDetailsModal.jsx @@ -12,12 +12,12 @@ dayjs.extend(utc); const UploadDetailsModal = (props) => { const { - onCancel, file, extraFields, onDownload, onRetry, onDelete, + onCancel, data, extraFields, onDownload, onRetry, onDelete, } = props; const { name, upload, size, lastModified, fileObject = undefined, - } = file ?? {}; + } = data ?? {}; const { progress, status } = upload ?? false; @@ -135,7 +135,7 @@ const UploadDetailsModal = (props) => { UploadDetailsModal.propTypes = { onCancel: PropTypes.func.isRequired, - file: PropTypes.object.isRequired, + data: PropTypes.object.isRequired, extraFields: PropTypes.object, onDownload: PropTypes.func.isRequired, onDelete: PropTypes.func.isRequired, diff --git a/src/components/data-management/metadata/CellLevelUploadModal.jsx b/src/components/data-management/metadata/CellLevelUploadModal.jsx index 0f586822f1..939911126b 100644 --- a/src/components/data-management/metadata/CellLevelUploadModal.jsx +++ b/src/components/data-management/metadata/CellLevelUploadModal.jsx @@ -272,7 +272,7 @@ const CellLevelUploadModal = (props) => { onDownload={downloadData} onDelete={() => dispatch(deleteCellLevelMetadata(activeExperimentId))} onRetry={() => onUploadFile(file)} - file={fileInfoObject} + data={fileInfoObject} /> ); } From 7f0e867a3e54a23b20f48aa231bc92b824f8096b Mon Sep 17 00:00:00 2001 From: cosa65 Date: Wed, 29 Nov 2023 11:21:00 -0300 Subject: [PATCH 05/53] Remove name and lastModified from sample files in redux Signed-off-by: cosa65 --- src/components/data-management/UploadDetailsModal.jsx | 4 +--- src/redux/actions/samples/loadSamples.js | 1 - src/redux/reducers/samples/samplesFileUpdate.js | 1 - 3 files changed, 1 insertion(+), 5 deletions(-) diff --git a/src/components/data-management/UploadDetailsModal.jsx b/src/components/data-management/UploadDetailsModal.jsx index 0f5fa1991e..671cf98698 100644 --- a/src/components/data-management/UploadDetailsModal.jsx +++ b/src/components/data-management/UploadDetailsModal.jsx @@ -16,7 +16,7 @@ const UploadDetailsModal = (props) => { } = props; const { - name, upload, size, lastModified, fileObject = undefined, + upload, size, lastModified, fileObject = undefined, } = data ?? {}; const { progress, status } = upload ?? false; @@ -119,8 +119,6 @@ const UploadDetailsModal = (props) => { )} {renderFields(extraFields)} - {!isNotUploadedModal && renderFields({ Filename: name })} - { isSuccessModal || isUploading ? renderFields({ 'File size': bytesToSize(size), 'Upload date': fromISODateToFormatted(lastModified) }) : renderFields({ Error: messageForStatus(status) }) diff --git a/src/redux/actions/samples/loadSamples.js b/src/redux/actions/samples/loadSamples.js index 8e644ff754..4a4c72d231 100644 --- a/src/redux/actions/samples/loadSamples.js +++ b/src/redux/actions/samples/loadSamples.js @@ -23,7 +23,6 @@ const toApiV1 = (samples, experimentId) => { apiV1Files[fileName] = { size: files[key].size, valid: true, - name: fileName, upload: { status: files[key].uploadStatus, }, diff --git a/src/redux/reducers/samples/samplesFileUpdate.js b/src/redux/reducers/samples/samplesFileUpdate.js index 73bf40b805..0488094194 100644 --- a/src/redux/reducers/samples/samplesFileUpdate.js +++ b/src/redux/reducers/samples/samplesFileUpdate.js @@ -26,7 +26,6 @@ const samplesFileUpdate = (state, action) => { ...state[sampleUuid].files, [fileName]: { ...newFile, - lastModified, }, }, lastModified, From 75f1a932fb8ede1a71e2732ae55692ba765f2bf6 Mon Sep 17 00:00:00 2001 From: cosa65 Date: Fri, 1 Dec 2023 12:20:35 -0300 Subject: [PATCH 06/53] Remove name from file in redux Signed-off-by: cosa65 --- .../data-management/FileUploadModal.jsx | 15 ++++-- src/redux/actions/samples/createSampleFile.js | 23 ++++++++- src/redux/actions/samples/createSamples.js | 17 ++++--- src/utils/upload/processUpload.js | 50 ++++++++++--------- 4 files changed, 71 insertions(+), 34 deletions(-) diff --git a/src/components/data-management/FileUploadModal.jsx b/src/components/data-management/FileUploadModal.jsx index 1469d0ee79..7f08acc0dd 100644 --- a/src/components/data-management/FileUploadModal.jsx +++ b/src/components/data-management/FileUploadModal.jsx @@ -22,7 +22,7 @@ import config from 'config'; import { sampleTech } from 'utils/constants'; import techOptions, { techNamesToDisplay } from 'utils/upload/fileUploadSpecifications'; import handleError from 'utils/http/handleError'; -import { fileObjectToFileRecord } from 'utils/upload/processUpload'; +import { fileObjectToFileRecord, getFileSampleAndName } from 'utils/upload/processUpload'; import integrationTestConstants from 'utils/integrationTestConstants'; import endUserMessages from 'utils/endUserMessages'; @@ -169,6 +169,14 @@ const FileUploadModal = (props) => { ); + const getFilePathToDisplay = (fileObject) => ( + // if the file has a path, trim to just the file and its folder. + // otherwise simply use its name + fileObject.path + ? Object.values(getFileSampleAndName(fileObject.path)).join('/') + : fileObject.name + ); + return ( { defaultValue={selectedTech} disabled={currentSelectedTech} onChange={(value) => setSelectedTech(value)} - style={{ width: 180 }} // Fix the width so that the dropdown doesn't change size when the value changes + // Fix the width so that the dropdown doesn't change size when the value changes + style={{ width: 180 }} > { Object.values(sampleTech) @@ -314,7 +323,7 @@ const FileUploadModal = (props) => { ellipsis={{ tooltip: file.name }} style={{ width: '200px' }} > - {file.name} + {getFilePathToDisplay(file.fileObject)} { removeFile(file.name); }} /> diff --git a/src/redux/actions/samples/createSampleFile.js b/src/redux/actions/samples/createSampleFile.js index ab89878115..68312129d9 100644 --- a/src/redux/actions/samples/createSampleFile.js +++ b/src/redux/actions/samples/createSampleFile.js @@ -26,6 +26,26 @@ const createSampleFile = ( size: fileForApiV1.size, }; + const { + size, + // valid: true, + upload, + // name: 'features.tsv.gz', + fileObject, + path, + errors, + compressed, + } = fileForApiV1; + + const fileForRedux = { + size, + upload, + fileObject, + path, + errors, + compressed, + }; + dispatch({ type: SAMPLES_FILE_UPDATE, payload: { @@ -33,7 +53,8 @@ const createSampleFile = ( lastModified: updatedAt, fileName: fileNameForApiV1[type], fileDiff: { - ...fileForApiV1, + // ...fileForApiV1, + ...fileForRedux, upload: { status: UploadStatus.UPLOADING, progress: 0, abortController, }, diff --git a/src/redux/actions/samples/createSamples.js b/src/redux/actions/samples/createSamples.js index ccbad5ab71..e2efdd3b31 100644 --- a/src/redux/actions/samples/createSamples.js +++ b/src/redux/actions/samples/createSamples.js @@ -15,7 +15,6 @@ import { sampleTech } from 'utils/constants'; import UploadStatus from 'utils/upload/UploadStatus'; import fileNameForApiV1 from 'utils/upload/fileNameForApiV1'; -import getFileTypeV2 from 'utils/getFileTypeV2'; // If the sample name of new samples coincides with already existing // ones we should not create new samples, @@ -116,17 +115,23 @@ const createSamples = ( createdDate, lastModified: createdDate, options, - metadata: experiment?.metadataKeys - .reduce((acc, curr) => ({ ...acc, [curr]: METADATA_DEFAULT_VALUE }), {}) || {}, - files: Object.values(files).reduce(((acc, curr) => { - const fileType = fileNameForApiV1[getFileTypeV2(curr.name, sampleTechnology)]; + metadata: experiment?.metadataKeys.reduce( + (acc, metadataKey) => ( + { ...acc, [metadataKey]: METADATA_DEFAULT_VALUE }), {}, + ) ?? {}, + + files: Object.values(files).reduce(((acc, file) => { + const fileTypeV1 = fileNameForApiV1[file.type]; return ( - { ...acc, [fileType]: { upload: { status: UploadStatus.UPLOADING } } } + { ...acc, [fileTypeV1]: { upload: { status: UploadStatus.UPLOADING } } } ); }), {}), })); + console.log('newSamplesToReduxDebug'); + console.log(newSamplesToRedux); + dispatch({ type: SAMPLES_CREATED, payload: { diff --git a/src/utils/upload/processUpload.js b/src/utils/upload/processUpload.js index 85b0924c7f..5be47390cc 100644 --- a/src/utils/upload/processUpload.js +++ b/src/utils/upload/processUpload.js @@ -64,8 +64,6 @@ const prepareAndUploadFileToS3 = async ( }; const createAndUploadSampleFile = async (file, experimentId, sampleId, dispatch, selectedTech) => { - const fileType = getFileTypeV2(file.name, selectedTech); - const abortController = new AbortController(); let sampleFileId; @@ -75,7 +73,7 @@ const createAndUploadSampleFile = async (file, experimentId, sampleId, dispatch, createSampleFile( experimentId, sampleId, - fileType, + file.type, file, abortController, ), @@ -85,11 +83,14 @@ const createAndUploadSampleFile = async (file, experimentId, sampleId, dispatch, return; } + // Take the fileName now because after loadAndCompressIfNecessary the name could be lost + const fileName = file.fileObject.name; + if (!file.compressed) { try { file.fileObject = await loadAndCompressIfNecessary(file, () => { dispatch(updateSampleFileUpload( - experimentId, sampleId, sampleFileId, fileType, UploadStatus.COMPRESSING, + experimentId, sampleId, sampleFileId, file.type, UploadStatus.COMPRESSING, )); }); @@ -100,7 +101,7 @@ const createAndUploadSampleFile = async (file, experimentId, sampleId, dispatch, : UploadStatus.FILE_READ_ERROR; dispatch(updateSampleFileUpload( - experimentId, sampleId, sampleFileId, fileType, fileErrorStatus, + experimentId, sampleId, sampleFileId, file.type, fileErrorStatus, )); return; } @@ -111,20 +112,22 @@ const createAndUploadSampleFile = async (file, experimentId, sampleId, dispatch, experimentId, sampleFileId, file.size, - getMetadata(file, selectedTech), + getMetadata(fileName, selectedTech), ); const updateSampleFileUploadProgress = (status, percentProgress = 0) => dispatch( updateSampleFileUpload( - experimentId, sampleId, sampleFileId, fileType, status, percentProgress, + experimentId, sampleId, sampleFileId, file.type, status, percentProgress, ), ); const uploadUrlParams = { signedUrls, uploadId, fileId: sampleFileId }; await prepareAndUploadFileToS3(file, uploadUrlParams, 'sample', abortController, updateSampleFileUploadProgress); } catch (e) { + console.log('eDebug'); + console.log(e); dispatch(updateSampleFileUpload( - experimentId, sampleId, sampleFileId, fileType, UploadStatus.UPLOAD_ERROR, + experimentId, sampleId, sampleFileId, file.type, UploadStatus.UPLOAD_ERROR, )); } }; @@ -140,29 +143,34 @@ const beginSampleFileUpload = async (experimentId, sampleFileId, size, metadata) }, ); -const getMetadata = (file, selectedTech) => { +const getMetadata = (fileName, selectedTech) => { const metadata = {}; if (selectedTech === sampleTech['10X']) { - if (file.name.includes('genes')) { + if (fileName.includes('genes')) { metadata.cellranger_version = 'v2'; - } else if (file.name.includes('features')) { + } else if (fileName.includes('features')) { metadata.cellranger_version = 'v3'; } } + return metadata; }; +const getFileSampleAndName = (filePath) => { + const [sample, name] = _.takeRight(filePath.split('/'), 2); + + return { sample, name }; +}; + const processUpload = async (filesList, technology, samples, experimentId, dispatch) => { // First use map to make it easy to add files in the already existing sample entry const samplesMap = filesList.reduce((acc, file) => { - const pathToArray = file.name.trim().replace(/[\s]{2,}/ig, ' ').split('/'); + const { sample: sampleName, name } = getFileSampleAndName(file.fileObject.path.replace(/[\s]{2,}/ig, ' ')); - const sampleName = pathToArray[0]; - const fileName = fileUploadSpecifications[technology].getCorrespondingName(_.last(pathToArray)); + const fileName = fileUploadSpecifications[technology].getCorrespondingName(name); - // Update the file name so that instead of being saved as - // e.g. WT13/matrix.tsv.gz, we save it as matrix.tsv.gz - file.name = fileName; + // TODO decide what to do with file.type before merging + file.type = getFileTypeV2(fileName, technology); const sampleUuid = Object.values(samples).filter( (s) => s.name === sampleName @@ -238,12 +246,6 @@ const processUpload = async (filesList, technology, samples, experimentId, dispa const fileObjectToFileRecord = async (fileObject, technology) => { // This is the first stage in uploading a file. - // if the file has a path, trim to just the file and its folder. - // otherwise simply use its name - const filename = (fileObject.path) - ? _.takeRight(fileObject.path.split('/'), 2).join('/') - : fileObject.name; - const verdict = await inspectFile(fileObject, technology); let error = ''; @@ -254,7 +256,6 @@ const fileObjectToFileRecord = async (fileObject, technology) => { } return { - name: filename, fileObject, size: fileObject.size, path: fileObject.path, @@ -272,6 +273,7 @@ export { fileObjectToFileRecord, createAndUploadSampleFile, prepareAndUploadFileToS3, + getFileSampleAndName, }; export default processUpload; From 49a7bc8e9fba56328bc5e3628c39a49269db1615 Mon Sep 17 00:00:00 2001 From: cosa65 Date: Tue, 5 Dec 2023 09:10:51 -0300 Subject: [PATCH 07/53] Update downloadSampleFile so that it still works Signed-off-by: cosa65 --- src/components/data-management/SamplesTableCells.jsx | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/components/data-management/SamplesTableCells.jsx b/src/components/data-management/SamplesTableCells.jsx index 240294ee1a..069037b3c8 100644 --- a/src/components/data-management/SamplesTableCells.jsx +++ b/src/components/data-management/SamplesTableCells.jsx @@ -127,7 +127,9 @@ const UploadCell = (props) => { } }; const onDownload = () => { - downloadSampleFile(activeExperimentId, sampleUuid, uploadDetailsModalData.name, selectedTech); + downloadSampleFile( + activeExperimentId, sampleUuid, uploadDetailsModalData.fileCategory, selectedTech, + ); }; const onUpload = (fileObject, retryUpload = false) => { From 4f1c5dcfcee30ac9250d5a0451b258418b980689 Mon Sep 17 00:00:00 2001 From: cosa65 Date: Tue, 5 Dec 2023 09:21:28 -0300 Subject: [PATCH 08/53] Fix, add valid again Signed-off-by: cosa65 --- src/redux/actions/samples/createSampleFile.js | 2 +- src/redux/actions/samples/createSamples.js | 3 --- src/utils/upload/processUpload.js | 2 -- 3 files changed, 1 insertion(+), 6 deletions(-) diff --git a/src/redux/actions/samples/createSampleFile.js b/src/redux/actions/samples/createSampleFile.js index 68312129d9..91052ecf56 100644 --- a/src/redux/actions/samples/createSampleFile.js +++ b/src/redux/actions/samples/createSampleFile.js @@ -28,7 +28,6 @@ const createSampleFile = ( const { size, - // valid: true, upload, // name: 'features.tsv.gz', fileObject, @@ -38,6 +37,7 @@ const createSampleFile = ( } = fileForApiV1; const fileForRedux = { + valid: true, size, upload, fileObject, diff --git a/src/redux/actions/samples/createSamples.js b/src/redux/actions/samples/createSamples.js index e2efdd3b31..e9d058a2c8 100644 --- a/src/redux/actions/samples/createSamples.js +++ b/src/redux/actions/samples/createSamples.js @@ -129,9 +129,6 @@ const createSamples = ( }), {}), })); - console.log('newSamplesToReduxDebug'); - console.log(newSamplesToRedux); - dispatch({ type: SAMPLES_CREATED, payload: { diff --git a/src/utils/upload/processUpload.js b/src/utils/upload/processUpload.js index 5be47390cc..b6c45fc0c8 100644 --- a/src/utils/upload/processUpload.js +++ b/src/utils/upload/processUpload.js @@ -124,8 +124,6 @@ const createAndUploadSampleFile = async (file, experimentId, sampleId, dispatch, const uploadUrlParams = { signedUrls, uploadId, fileId: sampleFileId }; await prepareAndUploadFileToS3(file, uploadUrlParams, 'sample', abortController, updateSampleFileUploadProgress); } catch (e) { - console.log('eDebug'); - console.log(e); dispatch(updateSampleFileUpload( experimentId, sampleId, sampleFileId, file.type, UploadStatus.UPLOAD_ERROR, )); From f4df7e4898e586151ecee967de5c7b57d021b77e Mon Sep 17 00:00:00 2001 From: cosa65 Date: Tue, 5 Dec 2023 09:54:25 -0300 Subject: [PATCH 09/53] Remove fileNameForApiV1 and replace its usages with the same type as is being used in sql Signed-off-by: cosa65 --- src/redux/actions/samples/createSampleFile.js | 3 +-- src/redux/actions/samples/createSamples.js | 12 +++--------- src/redux/actions/samples/loadSamples.js | 5 +---- src/redux/actions/samples/updateSampleFileUpload.js | 5 ++--- src/redux/reducers/samples/samplesFileUpdate.js | 6 +++--- src/utils/data-management/downloadSampleFile.js | 6 +----- src/utils/fileNamesByTech.js | 8 ++++---- src/utils/upload/fileNameForApiV1.js | 10 ---------- src/utils/upload/fileUploadSpecifications.js | 12 ++++++------ 9 files changed, 21 insertions(+), 46 deletions(-) delete mode 100644 src/utils/upload/fileNameForApiV1.js diff --git a/src/redux/actions/samples/createSampleFile.js b/src/redux/actions/samples/createSampleFile.js index 91052ecf56..de651af476 100644 --- a/src/redux/actions/samples/createSampleFile.js +++ b/src/redux/actions/samples/createSampleFile.js @@ -6,7 +6,6 @@ import { SAMPLES_FILE_UPDATE } from 'redux/actionTypes/samples'; import UploadStatus from 'utils/upload/UploadStatus'; import updateSampleFileUpload from 'redux/actions/samples/updateSampleFileUpload'; -import fileNameForApiV1 from 'utils/upload/fileNameForApiV1'; const createSampleFile = ( experimentId, @@ -50,8 +49,8 @@ const createSampleFile = ( type: SAMPLES_FILE_UPDATE, payload: { sampleUuid: sampleId, + sampleFileType: type, lastModified: updatedAt, - fileName: fileNameForApiV1[type], fileDiff: { // ...fileForApiV1, ...fileForRedux, diff --git a/src/redux/actions/samples/createSamples.js b/src/redux/actions/samples/createSamples.js index e9d058a2c8..aecdfc97fd 100644 --- a/src/redux/actions/samples/createSamples.js +++ b/src/redux/actions/samples/createSamples.js @@ -14,8 +14,6 @@ import { defaultSampleOptions, sampleTemplate } from 'redux/reducers/samples/ini import { sampleTech } from 'utils/constants'; import UploadStatus from 'utils/upload/UploadStatus'; -import fileNameForApiV1 from 'utils/upload/fileNameForApiV1'; - // If the sample name of new samples coincides with already existing // ones we should not create new samples, // just reuse their sampleIds and upload the new files @@ -120,13 +118,9 @@ const createSamples = ( { ...acc, [metadataKey]: METADATA_DEFAULT_VALUE }), {}, ) ?? {}, - files: Object.values(files).reduce(((acc, file) => { - const fileTypeV1 = fileNameForApiV1[file.type]; - - return ( - { ...acc, [fileTypeV1]: { upload: { status: UploadStatus.UPLOADING } } } - ); - }), {}), + files: Object.values(files).reduce(((acc, file) => ( + { ...acc, [file.type]: { upload: { status: UploadStatus.UPLOADING } } } + )), {}), })); dispatch({ diff --git a/src/redux/actions/samples/loadSamples.js b/src/redux/actions/samples/loadSamples.js index 4a4c72d231..3148d4b209 100644 --- a/src/redux/actions/samples/loadSamples.js +++ b/src/redux/actions/samples/loadSamples.js @@ -1,6 +1,5 @@ import fetchAPI from 'utils/http/fetchAPI'; import handleError from 'utils/http/handleError'; -import fileNameForApiV1 from 'utils/upload/fileNameForApiV1'; import { SAMPLES_LOADED, @@ -18,9 +17,7 @@ const toApiV1 = (samples, experimentId) => { const fileType = files[key]?.sampleFileType; if (!fileType) throw new Error('No sample file found'); - const fileName = fileNameForApiV1[fileType]; - - apiV1Files[fileName] = { + apiV1Files[fileType] = { size: files[key].size, valid: true, upload: { diff --git a/src/redux/actions/samples/updateSampleFileUpload.js b/src/redux/actions/samples/updateSampleFileUpload.js index c67d613434..4bb62d42ae 100644 --- a/src/redux/actions/samples/updateSampleFileUpload.js +++ b/src/redux/actions/samples/updateSampleFileUpload.js @@ -5,7 +5,6 @@ import endUserMessages from 'utils/endUserMessages'; import fetchAPI from 'utils/http/fetchAPI'; import handleError from 'utils/http/handleError'; import UploadStatus from 'utils/upload/UploadStatus'; -import fileNameForApiV1 from 'utils/upload/fileNameForApiV1'; const updateSampleFileUpload = ( experimentId, sampleId, sampleFileId, type, uploadStatus, uploadProgress, @@ -34,8 +33,8 @@ const updateSampleFileUpload = ( type: SAMPLES_FILE_UPDATE, payload: { sampleUuid: sampleId, + sampleFileType: type, lastModified: updatedAt, - fileName: fileNameForApiV1[type], fileDiff: { upload: { status: UploadStatus.UPLOAD_ERROR } }, }, }); @@ -50,8 +49,8 @@ const updateSampleFileUpload = ( type: SAMPLES_FILE_UPDATE, payload: { sampleUuid: sampleId, + sampleFileType: type, lastModified: updatedAt, - fileName: fileNameForApiV1[type], fileDiff: { upload: { status: uploadStatus, progress: uploadProgress } }, }, }); diff --git a/src/redux/reducers/samples/samplesFileUpdate.js b/src/redux/reducers/samples/samplesFileUpdate.js index 0488094194..befc77e855 100644 --- a/src/redux/reducers/samples/samplesFileUpdate.js +++ b/src/redux/reducers/samples/samplesFileUpdate.js @@ -2,7 +2,7 @@ import _ from 'lodash'; const samplesFileUpdate = (state, action) => { const { - sampleUuid, fileName, fileDiff, lastModified, + sampleUuid, sampleFileType, fileDiff, lastModified, } = action.payload; // There's a possible race condition where a file update can reach this place @@ -11,7 +11,7 @@ const samplesFileUpdate = (state, action) => { return state; } - const oldFile = state[sampleUuid].files?.[fileName]; + const oldFile = state[sampleUuid].files?.[sampleFileType]; let newFile = fileDiff; if (oldFile) { @@ -24,7 +24,7 @@ const samplesFileUpdate = (state, action) => { ...state[sampleUuid], files: { ...state[sampleUuid].files, - [fileName]: { + [sampleFileType]: { ...newFile, }, }, diff --git a/src/utils/data-management/downloadSampleFile.js b/src/utils/data-management/downloadSampleFile.js index 74807356ff..1ef1a09c42 100644 --- a/src/utils/data-management/downloadSampleFile.js +++ b/src/utils/data-management/downloadSampleFile.js @@ -1,11 +1,7 @@ import downloadFromUrl from 'utils/downloadFromUrl'; import fetchAPI from 'utils/http/fetchAPI'; -import getFileTypeV2 from 'utils/getFileTypeV2'; - -const downloadSampleFile = async (experimentId, sampleUuid, fileName, selectedTech) => { - const fileType = getFileTypeV2(fileName, selectedTech); - +const downloadSampleFile = async (experimentId, sampleUuid, fileType) => { const requestUrl = `/v2/experiments/${experimentId}/samples/${sampleUuid}/files/${fileType}/downloadUrl`; const downloadUrl = await fetchAPI(requestUrl); diff --git a/src/utils/fileNamesByTech.js b/src/utils/fileNamesByTech.js index 5a30475494..fb39638cc4 100644 --- a/src/utils/fileNamesByTech.js +++ b/src/utils/fileNamesByTech.js @@ -1,10 +1,10 @@ import { sampleTech } from 'utils/constants'; const fileNamesByTech = { - [sampleTech['10X']]: ['features.tsv.gz', 'barcodes.tsv.gz', 'matrix.mtx.gz'], - [sampleTech.H5]: ['matrix.h5.gz'], - [sampleTech.SEURAT]: ['r.rds'], - [sampleTech.RHAPSODY]: ['expression_data.st.gz'], + [sampleTech['10X']]: ['features10x', 'barcodes10x', 'matrix10x'], + [sampleTech.H5]: ['10x_h5'], + [sampleTech.SEURAT]: ['seurat'], + [sampleTech.RHAPSODY]: ['rhapsody'], }; export default fileNamesByTech; diff --git a/src/utils/upload/fileNameForApiV1.js b/src/utils/upload/fileNameForApiV1.js deleted file mode 100644 index 7031f96cb4..0000000000 --- a/src/utils/upload/fileNameForApiV1.js +++ /dev/null @@ -1,10 +0,0 @@ -const fileNameForApiV1 = { - matrix10x: 'matrix.mtx.gz', - barcodes10x: 'barcodes.tsv.gz', - features10x: 'features.tsv.gz', - seurat: 'r.rds', - rhapsody: 'expression_data.st.gz', - '10x_h5': 'matrix.h5.gz', -}; - -export default fileNameForApiV1; diff --git a/src/utils/upload/fileUploadSpecifications.js b/src/utils/upload/fileUploadSpecifications.js index 11eb4d7a25..a5aacaba24 100644 --- a/src/utils/upload/fileUploadSpecifications.js +++ b/src/utils/upload/fileUploadSpecifications.js @@ -32,9 +32,9 @@ const fileUploadSpecifications = { ['matrix.mtx or matrix.mtx.gz'], ], requiredFiles: [ - { key: 'barcodes.tsv.gz', displayedName: 'barcodes.tsv' }, - { key: 'features.tsv.gz', displayedName: 'genes.tsv' }, - { key: 'matrix.mtx.gz', displayedName: 'matrix.mtx' }, + { key: 'barcodes10x', displayedName: 'barcodes.tsv' }, + { key: 'features10x', displayedName: 'genes.tsv' }, + { key: 'matrix10x', displayedName: 'matrix.mtx' }, ], fileUploadParagraphs: [ 'For each sample, upload a folder containing the 3 required files. The folder\'s name will be used to name the sample in it. You can change this name later in Data Management.', @@ -62,7 +62,7 @@ const fileUploadSpecifications = { ['\uD83D\uDCA1if file size is over 15GB, try removing any assays not indicated above.'], ], requiredFiles: [ - { key: 'r.rds', displayedName: 'seurat rds' }, + { key: 'seurat', displayedName: 'seurat rds' }, ], fileUploadParagraphs: [ '

For your dataset, upload a single *.rds file with the Seurat object (max 15GB).

', @@ -80,7 +80,7 @@ const fileUploadSpecifications = { }, [sampleTech.RHAPSODY]: { acceptedFiles: new Set(['expression_data.st', 'expression_data.st.gz']), - requiredFiles: [{ key: 'expression_data.st.gz', displayedName: 'expression_data.st' }], + requiredFiles: [{ key: 'rhapsody', displayedName: 'expression_data.st' }], inputInfo: [ ['expression_data.st or expression_data.st.gz'], ], @@ -96,7 +96,7 @@ const fileUploadSpecifications = { }, [sampleTech.H5]: { acceptedFiles: new Set(['matrix.h5', 'matrix.h5.gz']), - requiredFiles: [{ key: 'matrix.h5.gz', displayedName: 'matrix.h5' }], + requiredFiles: [{ key: '10x_h5', displayedName: 'matrix.h5' }], inputInfo: [['matrix.h5 or matrix.h5.gz']], fileUploadParagraphs: [`For each sample, upload a folder containing the h5 file. The folder's name will be used to name the sample in it. From 761fffea9d19932af8e0bbc2521b4929d5343ef9 Mon Sep 17 00:00:00 2001 From: cosa65 Date: Tue, 5 Dec 2023 10:09:14 -0300 Subject: [PATCH 10/53] Remove valid bool from the sample files in redux, if a file is created in redux it is already validated so it will always be valid = true Signed-off-by: cosa65 --- src/components/data-management/LaunchAnalysisButton.jsx | 1 - src/redux/actions/samples/createSampleFile.js | 1 - src/redux/actions/samples/loadSamples.js | 1 - 3 files changed, 3 deletions(-) diff --git a/src/components/data-management/LaunchAnalysisButton.jsx b/src/components/data-management/LaunchAnalysisButton.jsx index 593d00f381..8fd8bc7cea 100644 --- a/src/components/data-management/LaunchAnalysisButton.jsx +++ b/src/components/data-management/LaunchAnalysisButton.jsx @@ -116,7 +116,6 @@ const LaunchAnalysisButton = () => { for (const fileName of fileNames) { const checkedFile = sample.files[fileName]; allUploaded = allUploaded - && checkedFile.valid && checkedFile.upload.status === UploadStatus.UPLOADED; if (!allUploaded) break; diff --git a/src/redux/actions/samples/createSampleFile.js b/src/redux/actions/samples/createSampleFile.js index de651af476..5f962b4b73 100644 --- a/src/redux/actions/samples/createSampleFile.js +++ b/src/redux/actions/samples/createSampleFile.js @@ -36,7 +36,6 @@ const createSampleFile = ( } = fileForApiV1; const fileForRedux = { - valid: true, size, upload, fileObject, diff --git a/src/redux/actions/samples/loadSamples.js b/src/redux/actions/samples/loadSamples.js index 3148d4b209..b9f6205ad8 100644 --- a/src/redux/actions/samples/loadSamples.js +++ b/src/redux/actions/samples/loadSamples.js @@ -19,7 +19,6 @@ const toApiV1 = (samples, experimentId) => { apiV1Files[fileType] = { size: files[key].size, - valid: true, upload: { status: files[key].uploadStatus, }, From ffd4db6d0961d7fbf1a1bd0266a928b850ab2b16 Mon Sep 17 00:00:00 2001 From: cosa65 Date: Tue, 5 Dec 2023 10:15:17 -0300 Subject: [PATCH 11/53] Remove check, it isn't necessary because sample doesnt get created if files ar emissing Signed-off-by: cosa65 --- .../data-management/LaunchAnalysisButton.jsx | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/src/components/data-management/LaunchAnalysisButton.jsx b/src/components/data-management/LaunchAnalysisButton.jsx index 8fd8bc7cea..17482a56ce 100644 --- a/src/components/data-management/LaunchAnalysisButton.jsx +++ b/src/components/data-management/LaunchAnalysisButton.jsx @@ -101,19 +101,10 @@ const LaunchAnalysisButton = () => { const metadataKeysAvailable = activeExperiment.metadataKeys.length; const allSampleFilesUploaded = (sample) => { - // Check if all files for a given tech have been uploaded - const fileNames = fileNamesByTech[sample.type]; - - if ( - !fileUploadSpecifications[sample.type].requiredFiles.every( - (file) => fileNames.includes(file.key), - ) - ) { return false; } - let allUploaded = true; // eslint-disable-next-line no-restricted-syntax - for (const fileName of fileNames) { + for (const fileName of Object.keys(sample.files)) { const checkedFile = sample.files[fileName]; allUploaded = allUploaded && checkedFile.upload.status === UploadStatus.UPLOADED; From 908e764527c0d8ab5a9b0277fb1bb9edee3497c8 Mon Sep 17 00:00:00 2001 From: cosa65 Date: Tue, 5 Dec 2023 10:48:48 -0300 Subject: [PATCH 12/53] Simplify allSampleFilesUploaded code Signed-off-by: cosa65 --- .../data-management/LaunchAnalysisButton.jsx | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/src/components/data-management/LaunchAnalysisButton.jsx b/src/components/data-management/LaunchAnalysisButton.jsx index 17482a56ce..dcb2d44cac 100644 --- a/src/components/data-management/LaunchAnalysisButton.jsx +++ b/src/components/data-management/LaunchAnalysisButton.jsx @@ -100,20 +100,9 @@ const LaunchAnalysisButton = () => { const metadataKeysAvailable = activeExperiment.metadataKeys.length; - const allSampleFilesUploaded = (sample) => { - let allUploaded = true; - - // eslint-disable-next-line no-restricted-syntax - for (const fileName of Object.keys(sample.files)) { - const checkedFile = sample.files[fileName]; - allUploaded = allUploaded - && checkedFile.upload.status === UploadStatus.UPLOADED; - - if (!allUploaded) break; - } - - return allUploaded; - }; + const allSampleFilesUploaded = (sample) => ( + Object.values(sample.files).every((file) => file.upload.status === UploadStatus.UPLOADED) + ); const allSampleMetadataInserted = (sample) => { if (!metadataKeysAvailable) return true; From ceb18ae73a507957dd4e0a8a3436f90cde1f269f Mon Sep 17 00:00:00 2001 From: cosa65 Date: Tue, 5 Dec 2023 11:22:51 -0300 Subject: [PATCH 13/53] Remove unused imports Signed-off-by: cosa65 --- src/components/data-management/LaunchAnalysisButton.jsx | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/components/data-management/LaunchAnalysisButton.jsx b/src/components/data-management/LaunchAnalysisButton.jsx index dcb2d44cac..b984bb7d5f 100644 --- a/src/components/data-management/LaunchAnalysisButton.jsx +++ b/src/components/data-management/LaunchAnalysisButton.jsx @@ -7,13 +7,11 @@ import _ from 'lodash'; import { modules, sampleTech } from 'utils/constants'; -import fileUploadSpecifications from 'utils/upload/fileUploadSpecifications'; import UploadStatus from 'utils/upload/UploadStatus'; import integrationTestConstants from 'utils/integrationTestConstants'; import { useAppRouter } from 'utils/AppRouteProvider'; import calculatePipelinesRerunStatus from 'utils/data-management/calculatePipelinesRerunStatus'; -import fileNamesByTech from 'utils/fileNamesByTech'; const LaunchButtonTemplate = (props) => { const { From 358c59028ef3d06c5d01dcd1478eac4713544129 Mon Sep 17 00:00:00 2001 From: cosa65 Date: Tue, 5 Dec 2023 11:28:02 -0300 Subject: [PATCH 14/53] Remove src/utils/fileNamesByTech.js, it is no longer in use Signed-off-by: cosa65 --- src/utils/fileNamesByTech.js | 10 ---------- 1 file changed, 10 deletions(-) delete mode 100644 src/utils/fileNamesByTech.js diff --git a/src/utils/fileNamesByTech.js b/src/utils/fileNamesByTech.js deleted file mode 100644 index fb39638cc4..0000000000 --- a/src/utils/fileNamesByTech.js +++ /dev/null @@ -1,10 +0,0 @@ -import { sampleTech } from 'utils/constants'; - -const fileNamesByTech = { - [sampleTech['10X']]: ['features10x', 'barcodes10x', 'matrix10x'], - [sampleTech.H5]: ['10x_h5'], - [sampleTech.SEURAT]: ['seurat'], - [sampleTech.RHAPSODY]: ['rhapsody'], -}; - -export default fileNamesByTech; From db818624b2939946017875d94745fcc11d3f1b75 Mon Sep 17 00:00:00 2001 From: cosa65 Date: Tue, 5 Dec 2023 12:05:38 -0300 Subject: [PATCH 15/53] Add fileTypeToDisplay and use to show the file category in the correct format Signed-off-by: cosa65 --- src/components/data-management/SamplesTableCells.jsx | 3 ++- src/utils/getFileTypeV2.js | 10 ++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/components/data-management/SamplesTableCells.jsx b/src/components/data-management/SamplesTableCells.jsx index 069037b3c8..732948127d 100644 --- a/src/components/data-management/SamplesTableCells.jsx +++ b/src/components/data-management/SamplesTableCells.jsx @@ -19,6 +19,7 @@ import downloadSampleFile from 'utils/data-management/downloadSampleFile'; import { createAndUploadSampleFile, fileObjectToFileRecord } from 'utils/upload/processUpload'; import endUserMessages from 'utils/endUserMessages'; import handleError from 'utils/http/handleError'; +import { fileTypeToDisplay } from 'utils/getFileTypeV2'; import EditableField from '../EditableField'; import UploadDetailsModal from './UploadDetailsModal'; @@ -166,7 +167,7 @@ const UploadCell = (props) => { onRetry={() => onUpload(uploadDetailsModalData, true)} extraFields={{ Sample: sample?.name, - Category: uploadDetailsModalData.fileCategory, + Category: fileTypeToDisplay[uploadDetailsModalData.fileCategory], }} /> )} diff --git a/src/utils/getFileTypeV2.js b/src/utils/getFileTypeV2.js index 4d9a09ede5..fb2f19a866 100644 --- a/src/utils/getFileTypeV2.js +++ b/src/utils/getFileTypeV2.js @@ -23,6 +23,15 @@ const fileTypesByTech = { }, }; +const fileTypeToDisplay = { + barcodes10x: 'barcodes.tsv', + features10x: 'genes.tsv', + matrix10x: 'matrix.mtx', + '10x_h5': 'matrix.h5', + seurat: 'seurat rds', + rhapsody: 'expression_data.st', +}; + const getFileTypeV2 = (fileName, selectedTech) => { const fileTypes = fileTypesByTech[selectedTech]; @@ -32,3 +41,4 @@ const getFileTypeV2 = (fileName, selectedTech) => { }; export default getFileTypeV2; +export { fileTypeToDisplay }; From 38a81c6963e242f61adb9830acb3da12bf312aa1 Mon Sep 17 00:00:00 2001 From: cosa65 Date: Tue, 5 Dec 2023 12:05:51 -0300 Subject: [PATCH 16/53] Remove commented out line Signed-off-by: cosa65 --- src/components/data-management/UploadDetailsModal.jsx | 1 - 1 file changed, 1 deletion(-) diff --git a/src/components/data-management/UploadDetailsModal.jsx b/src/components/data-management/UploadDetailsModal.jsx index 671cf98698..3f609fb289 100644 --- a/src/components/data-management/UploadDetailsModal.jsx +++ b/src/components/data-management/UploadDetailsModal.jsx @@ -25,7 +25,6 @@ const UploadDetailsModal = (props) => { const isNotUploadedModal = status === UploadStatus.FILE_NOT_FOUND; const isUploading = status === UploadStatus.UPLOADING; - // title={!isNotUploadedModal ? (isSuccessModal ? 'Upload successful' : 'Upload error') : 'File not found'} const modalTitle = messageForStatus(status); function bytesToSize(bytes) { From 90824a5850ed8b05cd21d0d3bfeb26ada7822a31 Mon Sep 17 00:00:00 2001 From: cosa65 Date: Wed, 6 Dec 2023 08:53:42 -0300 Subject: [PATCH 17/53] Rename file Signed-off-by: cosa65 --- src/components/data-management/SamplesTableCells.jsx | 2 +- src/utils/{getFileTypeV2.js => getFileType.js} | 4 ++-- src/utils/upload/processUpload.js | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) rename src/utils/{getFileTypeV2.js => getFileType.js} (92%) diff --git a/src/components/data-management/SamplesTableCells.jsx b/src/components/data-management/SamplesTableCells.jsx index 732948127d..2e16d611c3 100644 --- a/src/components/data-management/SamplesTableCells.jsx +++ b/src/components/data-management/SamplesTableCells.jsx @@ -19,7 +19,7 @@ import downloadSampleFile from 'utils/data-management/downloadSampleFile'; import { createAndUploadSampleFile, fileObjectToFileRecord } from 'utils/upload/processUpload'; import endUserMessages from 'utils/endUserMessages'; import handleError from 'utils/http/handleError'; -import { fileTypeToDisplay } from 'utils/getFileTypeV2'; +import { fileTypeToDisplay } from 'utils/getFileType'; import EditableField from '../EditableField'; import UploadDetailsModal from './UploadDetailsModal'; diff --git a/src/utils/getFileTypeV2.js b/src/utils/getFileType.js similarity index 92% rename from src/utils/getFileTypeV2.js rename to src/utils/getFileType.js index fb2f19a866..cabe4fe063 100644 --- a/src/utils/getFileTypeV2.js +++ b/src/utils/getFileType.js @@ -32,7 +32,7 @@ const fileTypeToDisplay = { rhapsody: 'expression_data.st', }; -const getFileTypeV2 = (fileName, selectedTech) => { +const getFileType = (fileName, selectedTech) => { const fileTypes = fileTypesByTech[selectedTech]; if (Object.keys(fileTypes).length === 1) return Object.values(fileTypes)[0]; @@ -40,5 +40,5 @@ const getFileTypeV2 = (fileName, selectedTech) => { return fileTypes[fileName]; }; -export default getFileTypeV2; +export default getFileType; export { fileTypeToDisplay }; diff --git a/src/utils/upload/processUpload.js b/src/utils/upload/processUpload.js index b6c45fc0c8..d709bc61e1 100644 --- a/src/utils/upload/processUpload.js +++ b/src/utils/upload/processUpload.js @@ -10,7 +10,7 @@ import loadAndCompressIfNecessary from 'utils/upload/loadAndCompressIfNecessary' import { inspectFile, Verdict } from 'utils/upload/fileInspector'; import fetchAPI from 'utils/http/fetchAPI'; -import getFileTypeV2 from 'utils/getFileTypeV2'; +import getFileType from 'utils/getFileType'; import { sampleTech } from 'utils/constants'; import fileUploadSpecifications from 'utils/upload/fileUploadSpecifications'; import processMultipartUpload from 'utils/upload/processMultipartUpload'; @@ -168,7 +168,7 @@ const processUpload = async (filesList, technology, samples, experimentId, dispa const fileName = fileUploadSpecifications[technology].getCorrespondingName(name); // TODO decide what to do with file.type before merging - file.type = getFileTypeV2(fileName, technology); + file.type = getFileType(fileName, technology); const sampleUuid = Object.values(samples).filter( (s) => s.name === sampleName From b6ac5ccbb2680fe412706fb7b3de4a0bf9caeff2 Mon Sep 17 00:00:00 2001 From: cosa65 Date: Wed, 6 Dec 2023 09:00:31 -0300 Subject: [PATCH 18/53] Simplify requiredFiles Signed-off-by: cosa65 --- src/components/data-management/SamplesTable.jsx | 13 +++++++------ src/utils/upload/fileUploadSpecifications.js | 14 ++++---------- 2 files changed, 11 insertions(+), 16 deletions(-) diff --git a/src/components/data-management/SamplesTable.jsx b/src/components/data-management/SamplesTable.jsx index a6275be0c0..61ba69b4a9 100644 --- a/src/components/data-management/SamplesTable.jsx +++ b/src/components/data-management/SamplesTable.jsx @@ -38,6 +38,7 @@ import integrationTestConstants from 'utils/integrationTestConstants'; import useConditionalEffect from 'utils/customHooks/useConditionalEffect'; import fileUploadSpecifications from 'utils/upload/fileUploadSpecifications'; import { sampleTech } from 'utils/constants'; +import { fileTypeToDisplay } from 'utils/getFileType'; const { Text } = Typography; @@ -90,19 +91,19 @@ const SamplesTable = forwardRef((props, ref) => { ), }, - ...fileUploadSpecifications[selectedTech]?.requiredFiles?.map((fileName, indx) => { - const fileNameWithoutExtension = fileName.key.split('.')[0]; + ...fileUploadSpecifications[selectedTech]?.requiredFiles?.map((requiredFile, indx) => { + const fileNameWithoutExtension = requiredFile.split('.')[0]; return ({ index: 2 + indx, - title:
{fileName.displayedName}
, + title:
{fileTypeToDisplay[requiredFile]}
, key: fileNameWithoutExtension, dataIndex: fileNameWithoutExtension, width: 170, onCell: () => ({ style: { margin: '0px', padding: '0px' } }), render: (tableCellData) => tableCellData && ( ), @@ -232,8 +233,8 @@ const SamplesTable = forwardRef((props, ref) => { const generateDataForItem = useCallback((sampleUuid) => { const sampleFileNames = fileUploadSpecifications[selectedTech]?.requiredFiles - .map((fileName) => ([ - fileName.key.split('.')[0], + .map((requiredFile) => ([ + requiredFile.split('.')[0], { sampleUuid }, ])); diff --git a/src/utils/upload/fileUploadSpecifications.js b/src/utils/upload/fileUploadSpecifications.js index a5aacaba24..9dc9eac12b 100644 --- a/src/utils/upload/fileUploadSpecifications.js +++ b/src/utils/upload/fileUploadSpecifications.js @@ -31,11 +31,7 @@ const fileUploadSpecifications = { ['barcodes.tsv or barcodes.tsv.gz'], ['matrix.mtx or matrix.mtx.gz'], ], - requiredFiles: [ - { key: 'barcodes10x', displayedName: 'barcodes.tsv' }, - { key: 'features10x', displayedName: 'genes.tsv' }, - { key: 'matrix10x', displayedName: 'matrix.mtx' }, - ], + requiredFiles: ['barcodes10x', 'features10x', 'matrix10x'], fileUploadParagraphs: [ 'For each sample, upload a folder containing the 3 required files. The folder\'s name will be used to name the sample in it. You can change this name later in Data Management.', 'The required files for each sample are:', @@ -61,9 +57,7 @@ const fileUploadSpecifications = { ['\uD83D\uDCA1sample level metadata in scdata@meta.data that groups samples in scdata$samples is auto-detected for downstream analysis.'], ['\uD83D\uDCA1if file size is over 15GB, try removing any assays not indicated above.'], ], - requiredFiles: [ - { key: 'seurat', displayedName: 'seurat rds' }, - ], + requiredFiles: ['seurat'], fileUploadParagraphs: [ '

For your dataset, upload a single *.rds file with the Seurat object (max 15GB).

', '

The Seurat object must contain the following slots and metadata:

', @@ -80,7 +74,7 @@ const fileUploadSpecifications = { }, [sampleTech.RHAPSODY]: { acceptedFiles: new Set(['expression_data.st', 'expression_data.st.gz']), - requiredFiles: [{ key: 'rhapsody', displayedName: 'expression_data.st' }], + requiredFiles: ['rhapsody'], inputInfo: [ ['expression_data.st or expression_data.st.gz'], ], @@ -96,7 +90,7 @@ const fileUploadSpecifications = { }, [sampleTech.H5]: { acceptedFiles: new Set(['matrix.h5', 'matrix.h5.gz']), - requiredFiles: [{ key: '10x_h5', displayedName: 'matrix.h5' }], + requiredFiles: ['10x_h5'], inputInfo: [['matrix.h5 or matrix.h5.gz']], fileUploadParagraphs: [`For each sample, upload a folder containing the h5 file. The folder's name will be used to name the sample in it. From adce0dd892f645663732cb1d476d5f104db6903b Mon Sep 17 00:00:00 2001 From: cosa65 Date: Wed, 6 Dec 2023 09:31:00 -0300 Subject: [PATCH 19/53] Switch to generating samplesMap with file types instead of the generic file names Signed-off-by: cosa65 --- src/utils/upload/processUpload.js | 2 +- src/utils/upload/validate10x.js | 36 ++++++++++------------------ src/utils/upload/validateRhapsody.js | 3 +-- 3 files changed, 15 insertions(+), 26 deletions(-) diff --git a/src/utils/upload/processUpload.js b/src/utils/upload/processUpload.js index d709bc61e1..fe7bcd696b 100644 --- a/src/utils/upload/processUpload.js +++ b/src/utils/upload/processUpload.js @@ -182,7 +182,7 @@ const processUpload = async (filesList, technology, samples, experimentId, dispa uuid: sampleUuid, files: { ...acc[sampleName]?.files, - [fileName]: file, + [file.type]: file, }, }, }; diff --git a/src/utils/upload/validate10x.js b/src/utils/upload/validate10x.js index 126b5cc419..8aff4938ac 100644 --- a/src/utils/upload/validate10x.js +++ b/src/utils/upload/validate10x.js @@ -74,22 +74,14 @@ const getMatrixHead = async (matrix) => { return matrixHeader; }; -const getSampleFiles = (sample) => { - const barcodes = sample.files['barcodes.tsv.gz'] || sample.files['barcodes.tsv']; - const features = sample.files['features.tsv.gz'] || sample.files['features.tsv'] || sample.files['genes.tsv.gz'] || sample.files['genes.tsv']; - const matrix = sample.files['matrix.mtx.gz'] || sample.files['matrix.mtx']; - - return { barcodes, features, matrix }; -}; - const validateSampleCompleteness = async (sampleFiles) => { - const { barcodes, features, matrix } = sampleFiles; + const { barcodes10x, features10x, matrix10x } = sampleFiles; const missingFiles = []; - if (!barcodes) missingFiles.push('barcodes'); - if (!features) missingFiles.push('features'); - if (!matrix) missingFiles.push('matrix'); + if (!barcodes10x) missingFiles.push('barcodes'); + if (!features10x) missingFiles.push('features'); + if (!matrix10x) missingFiles.push('matrix'); if (missingFiles.length) { throw new SampleValidationError(errorMessages.missingFiles(missingFiles)); @@ -97,9 +89,9 @@ const validateSampleCompleteness = async (sampleFiles) => { }; const validateMatrixFormat = async (sampleFiles) => { - const { matrix } = sampleFiles; + const { matrix10x } = sampleFiles; - const matrixHead = await getMatrixHead(matrix); + const matrixHead = await getMatrixHead(matrix10x); // Reject sample if type of count matrix is "array" not "coordinate" // See https://networkrepository.com/mtx-matrix-market-format.html @@ -110,18 +102,18 @@ const validateMatrixFormat = async (sampleFiles) => { }; const validateFileSizes = async (sampleFiles) => { - const { barcodes, features, matrix } = sampleFiles; + const { barcodes10x, features10x, matrix10x } = sampleFiles; const errors = []; - const matrixHead = await getMatrixHead(matrix); + const matrixHead = await getMatrixHead(matrix10x); const [ expectedNumFeatures, expectedNumBarcodes, ] = extractSampleSizes(matrixHead); - const numBarcodesFound = await getNumLines(barcodes); - const numFeaturesFound = await getNumLines(features); + const numBarcodesFound = await getNumLines(barcodes10x); + const numFeaturesFound = await getNumLines(features10x); if (numBarcodesFound === expectedNumFeatures && numFeaturesFound === expectedNumBarcodes) { @@ -144,11 +136,9 @@ const validateFileSizes = async (sampleFiles) => { }; const validate10x = async (sample) => { - const sampleFiles = getSampleFiles(sample); - - await validateSampleCompleteness(sampleFiles); - await validateMatrixFormat(sampleFiles); - await validateFileSizes(sampleFiles); + await validateSampleCompleteness(sample.files); + await validateMatrixFormat(sample.files); + await validateFileSizes(sample.files); }; export default validate10x; diff --git a/src/utils/upload/validateRhapsody.js b/src/utils/upload/validateRhapsody.js index 2badd2350e..c731ff26f3 100644 --- a/src/utils/upload/validateRhapsody.js +++ b/src/utils/upload/validateRhapsody.js @@ -6,8 +6,7 @@ const columnsToSearch = [ ]; const validateRhapsody = async (sample) => { - const fileObjectKey = Object.keys(sample.files).filter((key) => key.toLowerCase().includes('expression_data.st'))[0]; - const { compressed, fileObject } = sample.files[fileObjectKey]; + const { compressed, fileObject } = sample.files.rhapsody; const fileArrBuffer = await fileObject.slice(0, 800).arrayBuffer(); const file = compressed From 14d94166ad1b368fd9d8797d56fef4898acc9812 Mon Sep 17 00:00:00 2001 From: cosa65 Date: Wed, 6 Dec 2023 10:05:22 -0300 Subject: [PATCH 20/53] Move getFileType into getCorrespondingType Signed-off-by: cosa65 --- .../data-management/SamplesTable.jsx | 2 +- .../data-management/SamplesTableCells.jsx | 2 +- src/utils/fileTypeToDisplay.js | 10 +++++ src/utils/getFileType.js | 44 ------------------- src/utils/upload/fileUploadSpecifications.js | 23 +++++++--- src/utils/upload/processUpload.js | 6 +-- 6 files changed, 31 insertions(+), 56 deletions(-) create mode 100644 src/utils/fileTypeToDisplay.js delete mode 100644 src/utils/getFileType.js diff --git a/src/components/data-management/SamplesTable.jsx b/src/components/data-management/SamplesTable.jsx index 61ba69b4a9..b492b15cb3 100644 --- a/src/components/data-management/SamplesTable.jsx +++ b/src/components/data-management/SamplesTable.jsx @@ -38,7 +38,7 @@ import integrationTestConstants from 'utils/integrationTestConstants'; import useConditionalEffect from 'utils/customHooks/useConditionalEffect'; import fileUploadSpecifications from 'utils/upload/fileUploadSpecifications'; import { sampleTech } from 'utils/constants'; -import { fileTypeToDisplay } from 'utils/getFileType'; +import fileTypeToDisplay from 'utils/fileTypeToDisplay'; const { Text } = Typography; diff --git a/src/components/data-management/SamplesTableCells.jsx b/src/components/data-management/SamplesTableCells.jsx index 2e16d611c3..1d6e4a7c9a 100644 --- a/src/components/data-management/SamplesTableCells.jsx +++ b/src/components/data-management/SamplesTableCells.jsx @@ -19,7 +19,7 @@ import downloadSampleFile from 'utils/data-management/downloadSampleFile'; import { createAndUploadSampleFile, fileObjectToFileRecord } from 'utils/upload/processUpload'; import endUserMessages from 'utils/endUserMessages'; import handleError from 'utils/http/handleError'; -import { fileTypeToDisplay } from 'utils/getFileType'; +import fileTypeToDisplay from 'utils/fileTypeToDisplay'; import EditableField from '../EditableField'; import UploadDetailsModal from './UploadDetailsModal'; diff --git a/src/utils/fileTypeToDisplay.js b/src/utils/fileTypeToDisplay.js new file mode 100644 index 0000000000..51ef81d8d3 --- /dev/null +++ b/src/utils/fileTypeToDisplay.js @@ -0,0 +1,10 @@ +const fileTypeToDisplay = { + barcodes10x: 'barcodes.tsv', + features10x: 'genes.tsv', + matrix10x: 'matrix.mtx', + '10x_h5': 'matrix.h5', + seurat: 'seurat rds', + rhapsody: 'expression_data.st', +}; + +export default fileTypeToDisplay; diff --git a/src/utils/getFileType.js b/src/utils/getFileType.js deleted file mode 100644 index cabe4fe063..0000000000 --- a/src/utils/getFileType.js +++ /dev/null @@ -1,44 +0,0 @@ -import { sampleTech } from 'utils/constants'; - -const fileTypesByTech = { - [sampleTech['10X']]: { - // This handling won't be necessary after the file validation is refactored - 'matrix.mtx.gz': 'matrix10x', - 'barcodes.tsv.gz': 'barcodes10x', - 'features.tsv.gz': 'features10x', - 'genes.tsv.gz': 'features10x', - 'matrix.mtx': 'matrix10x', - 'barcodes.tsv': 'barcodes10x', - 'features.tsv': 'features10x', - 'genes.tsv': 'features10x', - }, - [sampleTech.H5]: { - 'matrix.h5': '10x_h5', - }, - [sampleTech.SEURAT]: { - 'r.rds': 'seurat', - }, - [sampleTech.RHAPSODY]: { - expression_data: 'rhapsody', - }, -}; - -const fileTypeToDisplay = { - barcodes10x: 'barcodes.tsv', - features10x: 'genes.tsv', - matrix10x: 'matrix.mtx', - '10x_h5': 'matrix.h5', - seurat: 'seurat rds', - rhapsody: 'expression_data.st', -}; - -const getFileType = (fileName, selectedTech) => { - const fileTypes = fileTypesByTech[selectedTech]; - - if (Object.keys(fileTypes).length === 1) return Object.values(fileTypes)[0]; - - return fileTypes[fileName]; -}; - -export default getFileType; -export { fileTypeToDisplay }; diff --git a/src/utils/upload/fileUploadSpecifications.js b/src/utils/upload/fileUploadSpecifications.js index 9dc9eac12b..a67a4526ca 100644 --- a/src/utils/upload/fileUploadSpecifications.js +++ b/src/utils/upload/fileUploadSpecifications.js @@ -40,10 +40,23 @@ const fileUploadSpecifications = { // setting to empty string allows folder upload on dropzone click webkitdirectory: '', isNameValid(fileName) { return matchFileName(fileName, this.acceptedFiles); }, - getCorrespondingName(fileName) { + getCorrespondingType(fileName) { + const fileNameToType = { + 'matrix.mtx.gz': 'matrix10x', + 'barcodes.tsv.gz': 'barcodes10x', + 'features.tsv.gz': 'features10x', + 'genes.tsv.gz': 'features10x', + 'matrix.mtx': 'matrix10x', + 'barcodes.tsv': 'barcodes10x', + 'features.tsv': 'features10x', + 'genes.tsv': 'features10x', + }; + const allowedNames = Array.from(this.acceptedFiles); - return allowedNames.find((allowedName) => fileName.endsWith(allowedName)); + const name = allowedNames.find((allowedName) => fileName.endsWith(allowedName)); + + return fileNameToType[name]; }, }, [sampleTech.SEURAT]: { @@ -70,7 +83,7 @@ const fileUploadSpecifications = { (validExtension) => fileName.endsWith(validExtension), ); }, - getCorrespondingName: () => 'r.rds', + getCorrespondingType: () => 'seurat', }, [sampleTech.RHAPSODY]: { acceptedFiles: new Set(['expression_data.st', 'expression_data.st.gz']), @@ -86,7 +99,7 @@ const fileUploadSpecifications = { dropzoneText: 'Drag and drop folders here or click to browse.', webkitdirectory: '', isNameValid: (fileName) => fileName.toLowerCase().match(/.*expression_data.st(.gz)?$/), - getCorrespondingName: (fileName) => fileName, + getCorrespondingType: () => 'rhapsody', }, [sampleTech.H5]: { acceptedFiles: new Set(['matrix.h5', 'matrix.h5.gz']), @@ -96,7 +109,7 @@ const fileUploadSpecifications = { name will be used to name the sample in it. You can change this name later in Data Management.`], isNameValid: (fileName) => fileName.toLowerCase().match(/.*matrix.h5(.gz)?$/), - getCorrespondingName: (fileName) => fileName, + getCorrespondingType: () => 'rhapsody', }, }; diff --git a/src/utils/upload/processUpload.js b/src/utils/upload/processUpload.js index fe7bcd696b..e5d99f2974 100644 --- a/src/utils/upload/processUpload.js +++ b/src/utils/upload/processUpload.js @@ -10,7 +10,6 @@ import loadAndCompressIfNecessary from 'utils/upload/loadAndCompressIfNecessary' import { inspectFile, Verdict } from 'utils/upload/fileInspector'; import fetchAPI from 'utils/http/fetchAPI'; -import getFileType from 'utils/getFileType'; import { sampleTech } from 'utils/constants'; import fileUploadSpecifications from 'utils/upload/fileUploadSpecifications'; import processMultipartUpload from 'utils/upload/processMultipartUpload'; @@ -165,10 +164,7 @@ const processUpload = async (filesList, technology, samples, experimentId, dispa const samplesMap = filesList.reduce((acc, file) => { const { sample: sampleName, name } = getFileSampleAndName(file.fileObject.path.replace(/[\s]{2,}/ig, ' ')); - const fileName = fileUploadSpecifications[technology].getCorrespondingName(name); - - // TODO decide what to do with file.type before merging - file.type = getFileType(fileName, technology); + file.type = fileUploadSpecifications[technology].getCorrespondingType(name); const sampleUuid = Object.values(samples).filter( (s) => s.name === sampleName From e1673656957b25b39638b33e77617fc508ea8ea0 Mon Sep 17 00:00:00 2001 From: cosa65 Date: Wed, 6 Dec 2023 10:10:07 -0300 Subject: [PATCH 21/53] Remove requiredFile.split('.')[0] on requiredFiles, it is no longer necessary Signed-off-by: cosa65 --- .../data-management/SamplesTable.jsx | 37 ++++++++----------- 1 file changed, 15 insertions(+), 22 deletions(-) diff --git a/src/components/data-management/SamplesTable.jsx b/src/components/data-management/SamplesTable.jsx index b492b15cb3..c7e293dd3a 100644 --- a/src/components/data-management/SamplesTable.jsx +++ b/src/components/data-management/SamplesTable.jsx @@ -91,24 +91,20 @@ const SamplesTable = forwardRef((props, ref) => { ), }, - ...fileUploadSpecifications[selectedTech]?.requiredFiles?.map((requiredFile, indx) => { - const fileNameWithoutExtension = requiredFile.split('.')[0]; - - return ({ - index: 2 + indx, - title:
{fileTypeToDisplay[requiredFile]}
, - key: fileNameWithoutExtension, - dataIndex: fileNameWithoutExtension, - width: 170, - onCell: () => ({ style: { margin: '0px', padding: '0px' } }), - render: (tableCellData) => tableCellData && ( - - ), - }); - }) || [], + ...fileUploadSpecifications[selectedTech]?.requiredFiles?.map((requiredFile, indx) => ({ + index: 2 + indx, + title:
{fileTypeToDisplay[requiredFile]}
, + key: requiredFile, + dataIndex: requiredFile, + width: 170, + onCell: () => ({ style: { margin: '0px', padding: '0px' } }), + render: (tableCellData) => tableCellData && ( + + ), + })) || [], ]), [selectedTech]); @@ -233,10 +229,7 @@ const SamplesTable = forwardRef((props, ref) => { const generateDataForItem = useCallback((sampleUuid) => { const sampleFileNames = fileUploadSpecifications[selectedTech]?.requiredFiles - .map((requiredFile) => ([ - requiredFile.split('.')[0], - { sampleUuid }, - ])); + .map((requiredFile) => ([requiredFile, { sampleUuid }])); return { key: sampleUuid, From 0bdbddea48809a44a59f32fc618ab948cb611e96 Mon Sep 17 00:00:00 2001 From: cosa65 Date: Wed, 6 Dec 2023 10:17:47 -0300 Subject: [PATCH 22/53] Rename file to fileObject Signed-off-by: cosa65 --- src/components/data-management/FileUploadModal.jsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/components/data-management/FileUploadModal.jsx b/src/components/data-management/FileUploadModal.jsx index 7f08acc0dd..7eff86592c 100644 --- a/src/components/data-management/FileUploadModal.jsx +++ b/src/components/data-management/FileUploadModal.jsx @@ -107,8 +107,8 @@ const FileUploadModal = (props) => { filteredFiles = filteredFiles // Remove all files that aren't in a folder - .filter((file) => { - const inFolder = file.path.includes('/'); + .filter((fileObject) => { + const inFolder = fileObject.path.includes('/'); filesNotInFolder ||= !inFolder; From 7a7eae9b42bc7101879efa903a5526ab94a0e517 Mon Sep 17 00:00:00 2001 From: cosa65 Date: Wed, 6 Dec 2023 11:49:46 -0300 Subject: [PATCH 23/53] Move matrix10x and other names into an enum Signed-off-by: cosa65 --- .../mockData/generateMockSamples.js | 7 ++++--- .../data-management/SamplesTable.jsx | 2 +- .../data-management/SamplesTableCells.jsx | 2 +- src/utils/fileTypeToDisplay.js | 10 --------- src/utils/sampleFileType.js | 21 +++++++++++++++++++ src/utils/upload/fileUploadSpecifications.js | 19 +++++++++-------- 6 files changed, 37 insertions(+), 24 deletions(-) delete mode 100644 src/utils/fileTypeToDisplay.js create mode 100644 src/utils/sampleFileType.js diff --git a/src/__test__/test-utils/mockData/generateMockSamples.js b/src/__test__/test-utils/mockData/generateMockSamples.js index 458868d210..39e3024ca7 100644 --- a/src/__test__/test-utils/mockData/generateMockSamples.js +++ b/src/__test__/test-utils/mockData/generateMockSamples.js @@ -1,6 +1,7 @@ import _ from 'lodash'; import fake from '__test__/test-utils/constants'; +import sampleFileType from 'utils/sampleFileType'; const mockSampleTemplate = (experimentId, sampleId, idx) => ({ id: sampleId, @@ -13,19 +14,19 @@ const mockSampleTemplate = (experimentId, sampleId, idx) => ({ files: { matrix10X: { uploadStatus: 'uploaded', - sampleFileType: 'matrix10x', + sampleFileType: sampleFileType.MATRIX_10_X, size: 1000, s3Path: 'testcfd8122f-25af-4f1a-a306-3268d44ed401', }, barcodes10X: { uploadStatus: 'uploaded', - sampleFileType: 'barcodes10x', + sampleFileType: sampleFileType.BARCODES_10_X, size: 100, s3Path: 'testcfd8122f-25af-4f1a-a306-3268d44ed402', }, features10X: { uploadStatus: 'uploaded', - sampleFileType: 'features10x', + sampleFileType: sampleFileType.FEATURES_10_X, size: 100, s3Path: 'testcfd8122f-25af-4f1a-a306-3268d44ed403', }, diff --git a/src/components/data-management/SamplesTable.jsx b/src/components/data-management/SamplesTable.jsx index c7e293dd3a..c456315091 100644 --- a/src/components/data-management/SamplesTable.jsx +++ b/src/components/data-management/SamplesTable.jsx @@ -38,7 +38,7 @@ import integrationTestConstants from 'utils/integrationTestConstants'; import useConditionalEffect from 'utils/customHooks/useConditionalEffect'; import fileUploadSpecifications from 'utils/upload/fileUploadSpecifications'; import { sampleTech } from 'utils/constants'; -import fileTypeToDisplay from 'utils/fileTypeToDisplay'; +import { fileTypeToDisplay } from 'utils/sampleFileType'; const { Text } = Typography; diff --git a/src/components/data-management/SamplesTableCells.jsx b/src/components/data-management/SamplesTableCells.jsx index 1d6e4a7c9a..a02bf5659f 100644 --- a/src/components/data-management/SamplesTableCells.jsx +++ b/src/components/data-management/SamplesTableCells.jsx @@ -19,7 +19,7 @@ import downloadSampleFile from 'utils/data-management/downloadSampleFile'; import { createAndUploadSampleFile, fileObjectToFileRecord } from 'utils/upload/processUpload'; import endUserMessages from 'utils/endUserMessages'; import handleError from 'utils/http/handleError'; -import fileTypeToDisplay from 'utils/fileTypeToDisplay'; +import { fileTypeToDisplay } from 'utils/sampleFileType'; import EditableField from '../EditableField'; import UploadDetailsModal from './UploadDetailsModal'; diff --git a/src/utils/fileTypeToDisplay.js b/src/utils/fileTypeToDisplay.js deleted file mode 100644 index 51ef81d8d3..0000000000 --- a/src/utils/fileTypeToDisplay.js +++ /dev/null @@ -1,10 +0,0 @@ -const fileTypeToDisplay = { - barcodes10x: 'barcodes.tsv', - features10x: 'genes.tsv', - matrix10x: 'matrix.mtx', - '10x_h5': 'matrix.h5', - seurat: 'seurat rds', - rhapsody: 'expression_data.st', -}; - -export default fileTypeToDisplay; diff --git a/src/utils/sampleFileType.js b/src/utils/sampleFileType.js new file mode 100644 index 0000000000..54798975c1 --- /dev/null +++ b/src/utils/sampleFileType.js @@ -0,0 +1,21 @@ +const sampleFileType = { + BARCODES_10_X: 'barcodes10x', + FEATURES_10_X: 'features10x', + MATRIX_10_X: 'matrix10x', + H5_10_X: '10x_h5', + SEURAT: 'seurat', + RHAPSODY: 'rhapsody', +}; + +const fileTypeToDisplay = { + [sampleFileType.BARCODES_10_X]: 'barcodes.tsv', + [sampleFileType.FEATURES_10_X]: 'genes.tsv', + [sampleFileType.MATRIX_10_X]: 'matrix.mtx', + [sampleFileType.H5_10_X]: 'matrix.h5', + [sampleFileType.SEURAT]: 'seurat rds', + [sampleFileType.RHAPSODY]: 'expression_data.st', +}; + +export default sampleFileType; + +export { fileTypeToDisplay }; diff --git a/src/utils/upload/fileUploadSpecifications.js b/src/utils/upload/fileUploadSpecifications.js index a67a4526ca..5d88db532c 100644 --- a/src/utils/upload/fileUploadSpecifications.js +++ b/src/utils/upload/fileUploadSpecifications.js @@ -1,4 +1,5 @@ import { sampleTech } from 'utils/constants'; +import sampleFileType from 'utils/sampleFileType'; const techNamesToDisplay = { [sampleTech['10X']]: '10X Chromium', @@ -31,7 +32,7 @@ const fileUploadSpecifications = { ['barcodes.tsv or barcodes.tsv.gz'], ['matrix.mtx or matrix.mtx.gz'], ], - requiredFiles: ['barcodes10x', 'features10x', 'matrix10x'], + requiredFiles: [sampleFileType.BARCODES_10_X, sampleFileType.FEATURES_10_X, sampleFileType.MATRIX_10_X], fileUploadParagraphs: [ 'For each sample, upload a folder containing the 3 required files. The folder\'s name will be used to name the sample in it. You can change this name later in Data Management.', 'The required files for each sample are:', @@ -42,14 +43,14 @@ const fileUploadSpecifications = { isNameValid(fileName) { return matchFileName(fileName, this.acceptedFiles); }, getCorrespondingType(fileName) { const fileNameToType = { - 'matrix.mtx.gz': 'matrix10x', - 'barcodes.tsv.gz': 'barcodes10x', - 'features.tsv.gz': 'features10x', - 'genes.tsv.gz': 'features10x', - 'matrix.mtx': 'matrix10x', - 'barcodes.tsv': 'barcodes10x', - 'features.tsv': 'features10x', - 'genes.tsv': 'features10x', + 'barcodes.tsv.gz': sampleFileType.BARCODES_10_X, + 'barcodes.tsv': sampleFileType.BARCODES_10_X, + 'features.tsv.gz': sampleFileType.FEATURES_10_X, + 'genes.tsv.gz': sampleFileType.FEATURES_10_X, + 'features.tsv': sampleFileType.FEATURES_10_X, + 'genes.tsv': sampleFileType.FEATURES_10_X, + 'matrix.mtx.gz': sampleFileType.MATRIX_10_X, + 'matrix.mtx': sampleFileType.MATRIX_10_X, }; const allowedNames = Array.from(this.acceptedFiles); From ba11b013da27726a35a45b475d017a26d4d148cf Mon Sep 17 00:00:00 2001 From: cosa65 Date: Wed, 6 Dec 2023 11:50:35 -0300 Subject: [PATCH 24/53] Cleanup Signed-off-by: cosa65 --- src/redux/actions/samples/createSampleFile.js | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/redux/actions/samples/createSampleFile.js b/src/redux/actions/samples/createSampleFile.js index 5f962b4b73..6afec25200 100644 --- a/src/redux/actions/samples/createSampleFile.js +++ b/src/redux/actions/samples/createSampleFile.js @@ -11,7 +11,7 @@ const createSampleFile = ( experimentId, sampleId, type, - fileForApiV1, + file, abortController, ) => async (dispatch) => { const updatedAt = dayjs().toISOString(); @@ -22,18 +22,17 @@ const createSampleFile = ( const url = `/v2/experiments/${experimentId}/samples/${sampleId}/sampleFiles/${type}`; const body = { sampleFileId, - size: fileForApiV1.size, + size: file.size, }; const { size, upload, - // name: 'features.tsv.gz', fileObject, path, errors, compressed, - } = fileForApiV1; + } = file; const fileForRedux = { size, @@ -51,7 +50,6 @@ const createSampleFile = ( sampleFileType: type, lastModified: updatedAt, fileDiff: { - // ...fileForApiV1, ...fileForRedux, upload: { status: UploadStatus.UPLOADING, progress: 0, abortController, From b63f0e2355378638eb3295e923176399b441ba76 Mon Sep 17 00:00:00 2001 From: cosa65 Date: Wed, 6 Dec 2023 11:52:46 -0300 Subject: [PATCH 25/53] Clean up Signed-off-by: cosa65 --- src/redux/actions/samples/loadSamples.js | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/src/redux/actions/samples/loadSamples.js b/src/redux/actions/samples/loadSamples.js index b9f6205ad8..1b125c3698 100644 --- a/src/redux/actions/samples/loadSamples.js +++ b/src/redux/actions/samples/loadSamples.js @@ -7,10 +7,10 @@ import { SAMPLES_LOADING, } from 'redux/actionTypes/samples'; -const toApiV1 = (samples, experimentId) => { - const apiV1Samples = {}; +const adaptedToRedux = (samples, experimentId) => { + const reduxSamples = {}; - const buildApiv1Files = (files) => { + const buildReduxFiles = (files) => { const apiV1Files = {}; Object.keys(files).forEach((key) => { @@ -29,21 +29,21 @@ const toApiV1 = (samples, experimentId) => { }; samples.forEach((sample) => { - const apiV1Files = buildApiv1Files(sample.files); - apiV1Samples[sample.id] = { + const reduxFiles = buildReduxFiles(sample.files); + reduxSamples[sample.id] = { experimentId, metadata: sample.metadata, createdDate: sample.createdAt, name: sample.name, lastModified: sample.updatedAt, - files: apiV1Files, + files: reduxFiles, type: sample.sampleTechnology, options: sample.options, uuid: sample.id, }; }); - return apiV1Samples; + return reduxSamples; }; const loadSamples = (experimentId) => async (dispatch) => { @@ -56,7 +56,7 @@ const loadSamples = (experimentId) => async (dispatch) => { const url = `/v2/experiments/${experimentId}/samples`; const data = await fetchAPI(url); - const samples = toApiV1(data, experimentId); + const samples = adaptedToRedux(data, experimentId); dispatch({ type: SAMPLES_LOADED, @@ -78,5 +78,3 @@ const loadSamples = (experimentId) => async (dispatch) => { }; export default loadSamples; - -export { toApiV1 }; From bbfda2f3fcdd084c9994470619e8bf4714a141ea Mon Sep 17 00:00:00 2001 From: cosa65 Date: Thu, 7 Dec 2023 09:02:23 -0300 Subject: [PATCH 26/53] Store in redux only necessary files Signed-off-by: cosa65 --- src/redux/actions/samples/createSampleFile.js | 22 +++++-------------- 1 file changed, 5 insertions(+), 17 deletions(-) diff --git a/src/redux/actions/samples/createSampleFile.js b/src/redux/actions/samples/createSampleFile.js index 6afec25200..56e3420427 100644 --- a/src/redux/actions/samples/createSampleFile.js +++ b/src/redux/actions/samples/createSampleFile.js @@ -1,3 +1,4 @@ +import _ from 'lodash'; import dayjs from 'dayjs'; import { v4 as uuidv4 } from 'uuid'; @@ -25,23 +26,10 @@ const createSampleFile = ( size: file.size, }; - const { - size, - upload, - fileObject, - path, - errors, - compressed, - } = file; - - const fileForRedux = { - size, - upload, - fileObject, - path, - errors, - compressed, - }; + // Leaving out path, errors, compressed, type, valid + // They are used during the upload process, not redux + // TODO we should check if they can be removed althogether from the file + const fileForRedux = _.pick(file, ['size', 'upload', 'fileObject']); dispatch({ type: SAMPLES_FILE_UPDATE, From e4fd9dc2b75afcc1e2bd99ec92b38c49b206ef06 Mon Sep 17 00:00:00 2001 From: cosa65 Date: Thu, 7 Dec 2023 09:07:18 -0300 Subject: [PATCH 27/53] Remove unreachable code path Signed-off-by: cosa65 --- .../data-management/SamplesTableCells.jsx | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/src/components/data-management/SamplesTableCells.jsx b/src/components/data-management/SamplesTableCells.jsx index a02bf5659f..977a1c6ba7 100644 --- a/src/components/data-management/SamplesTableCells.jsx +++ b/src/components/data-management/SamplesTableCells.jsx @@ -133,22 +133,9 @@ const UploadCell = (props) => { ); }; - const onUpload = (fileObject, retryUpload = false) => { - if (!uploadDetailsModalData) { - return; - } + const onRetry = (fileObject) => { // if retrying an upload we dont need to revalidate the file since it was done before - if (retryUpload) { - createAndUploadSampleFile(fileObject, activeExperimentId, sampleUuid, dispatch, selectedTech); - } else { - fileObjectToFileRecord(fileObject, selectedTech).then((newFile) => { - if (newFile.valid) { - createAndUploadSampleFile(newFile, activeExperimentId, sampleUuid, dispatch, selectedTech); - } else { - handleError('error', endUserMessages.ERROR_FILE_CATEGORY); - } - }); - } + createAndUploadSampleFile(fileObject, activeExperimentId, sampleUuid, dispatch, selectedTech); setUploadDetailsModalVisible(false); }; @@ -164,7 +151,7 @@ const UploadCell = (props) => { onCancel={() => setUploadDetailsModalVisible(false)} onDownload={onDownload} onDelete={() => dispatch(deleteSamples([sampleUuid]))} - onRetry={() => onUpload(uploadDetailsModalData, true)} + onRetry={() => onRetry(uploadDetailsModalData, true)} extraFields={{ Sample: sample?.name, Category: fileTypeToDisplay[uploadDetailsModalData.fileCategory], From 5209ab5e814df5da23fa6021e6eceea17b68c7f7 Mon Sep 17 00:00:00 2001 From: cosa65 Date: Thu, 7 Dec 2023 09:10:20 -0300 Subject: [PATCH 28/53] Remove file.valid, it is the opposite of file.errors so it just adds confusion Signed-off-by: cosa65 --- src/components/data-management/FileUploadModal.jsx | 4 ++-- src/redux/actions/samples/createSampleFile.js | 2 +- src/utils/upload/processUpload.js | 1 - 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/components/data-management/FileUploadModal.jsx b/src/components/data-management/FileUploadModal.jsx index 7eff86592c..818794d928 100644 --- a/src/components/data-management/FileUploadModal.jsx +++ b/src/components/data-management/FileUploadModal.jsx @@ -67,7 +67,7 @@ const FileUploadModal = (props) => { const [filesList, setFilesList] = useState([]); useEffect(() => { - setCanUpload(filesList.length && filesList.every((file) => file.valid)); + setCanUpload(filesList.length && filesList.every((file) => !file.errors)); }, [filesList]); useEffect(() => { @@ -309,7 +309,7 @@ const FileUploadModal = (props) => { style={{ width: '100%' }} > - {file.valid + {!file.errors ? ( <> diff --git a/src/redux/actions/samples/createSampleFile.js b/src/redux/actions/samples/createSampleFile.js index 56e3420427..eb027b087b 100644 --- a/src/redux/actions/samples/createSampleFile.js +++ b/src/redux/actions/samples/createSampleFile.js @@ -26,7 +26,7 @@ const createSampleFile = ( size: file.size, }; - // Leaving out path, errors, compressed, type, valid + // Leaving out path, errors, compressed, type // They are used during the upload process, not redux // TODO we should check if they can be removed althogether from the file const fileForRedux = _.pick(file, ['size', 'upload', 'fileObject']); diff --git a/src/utils/upload/processUpload.js b/src/utils/upload/processUpload.js index e5d99f2974..a9e4148d2c 100644 --- a/src/utils/upload/processUpload.js +++ b/src/utils/upload/processUpload.js @@ -257,7 +257,6 @@ const fileObjectToFileRecord = async (fileObject, technology) => { status: UploadStatus.UPLOADING, progress: 0, }, - valid: !error, errors: error, compressed: verdict === Verdict.VALID_ZIPPED, }; From ca2c14f52162ec86de7deccf26232e6e00e8292a Mon Sep 17 00:00:00 2001 From: cosa65 Date: Thu, 7 Dec 2023 10:02:12 -0300 Subject: [PATCH 29/53] Remove type from file, also fix retry upload Signed-off-by: cosa65 --- .../data-management/SamplesTableCells.jsx | 21 +++++++++++++----- src/redux/actions/samples/createSamples.js | 4 ++-- src/utils/upload/processUpload.js | 22 +++++++++++-------- 3 files changed, 30 insertions(+), 17 deletions(-) diff --git a/src/components/data-management/SamplesTableCells.jsx b/src/components/data-management/SamplesTableCells.jsx index 977a1c6ba7..7f71510da0 100644 --- a/src/components/data-management/SamplesTableCells.jsx +++ b/src/components/data-management/SamplesTableCells.jsx @@ -1,3 +1,4 @@ +import _ from 'lodash'; import React, { useEffect, useState } from 'react'; import { Space, Typography, Progress, Tooltip, Button, @@ -16,9 +17,7 @@ import integrationTestConstants from 'utils/integrationTestConstants'; import UploadStatus, { messageForStatus } from 'utils/upload/UploadStatus'; import styles from 'components/data-management/SamplesTableCells.module.css'; import downloadSampleFile from 'utils/data-management/downloadSampleFile'; -import { createAndUploadSampleFile, fileObjectToFileRecord } from 'utils/upload/processUpload'; -import endUserMessages from 'utils/endUserMessages'; -import handleError from 'utils/http/handleError'; +import { createAndUploadSampleFile } from 'utils/upload/processUpload'; import { fileTypeToDisplay } from 'utils/sampleFileType'; import EditableField from '../EditableField'; import UploadDetailsModal from './UploadDetailsModal'; @@ -133,9 +132,19 @@ const UploadCell = (props) => { ); }; - const onRetry = (fileObject) => { + const onRetry = () => { + const fileToUpload = _.pick(uploadDetailsModalData, ['size', 'upload', 'fileObject']); + const fileType = uploadDetailsModalData.fileCategory; + // if retrying an upload we dont need to revalidate the file since it was done before - createAndUploadSampleFile(fileObject, activeExperimentId, sampleUuid, dispatch, selectedTech); + createAndUploadSampleFile( + fileToUpload, + fileType, + activeExperimentId, + sampleUuid, + dispatch, + selectedTech, + ); setUploadDetailsModalVisible(false); }; @@ -151,7 +160,7 @@ const UploadCell = (props) => { onCancel={() => setUploadDetailsModalVisible(false)} onDownload={onDownload} onDelete={() => dispatch(deleteSamples([sampleUuid]))} - onRetry={() => onRetry(uploadDetailsModalData, true)} + onRetry={() => onRetry()} extraFields={{ Sample: sample?.name, Category: fileTypeToDisplay[uploadDetailsModalData.fileCategory], diff --git a/src/redux/actions/samples/createSamples.js b/src/redux/actions/samples/createSamples.js index aecdfc97fd..0cdd74f717 100644 --- a/src/redux/actions/samples/createSamples.js +++ b/src/redux/actions/samples/createSamples.js @@ -118,8 +118,8 @@ const createSamples = ( { ...acc, [metadataKey]: METADATA_DEFAULT_VALUE }), {}, ) ?? {}, - files: Object.values(files).reduce(((acc, file) => ( - { ...acc, [file.type]: { upload: { status: UploadStatus.UPLOADING } } } + files: Object.keys(files).reduce(((acc, [fileType]) => ( + { ...acc, [fileType]: { upload: { status: UploadStatus.UPLOADING } } } )), {}), })); diff --git a/src/utils/upload/processUpload.js b/src/utils/upload/processUpload.js index a9e4148d2c..121dd06f86 100644 --- a/src/utils/upload/processUpload.js +++ b/src/utils/upload/processUpload.js @@ -62,7 +62,9 @@ const prepareAndUploadFileToS3 = async ( return parts; }; -const createAndUploadSampleFile = async (file, experimentId, sampleId, dispatch, selectedTech) => { +const createAndUploadSampleFile = async ( + file, fileType, experimentId, sampleId, dispatch, selectedTech, +) => { const abortController = new AbortController(); let sampleFileId; @@ -72,7 +74,7 @@ const createAndUploadSampleFile = async (file, experimentId, sampleId, dispatch, createSampleFile( experimentId, sampleId, - file.type, + fileType, file, abortController, ), @@ -89,7 +91,7 @@ const createAndUploadSampleFile = async (file, experimentId, sampleId, dispatch, try { file.fileObject = await loadAndCompressIfNecessary(file, () => { dispatch(updateSampleFileUpload( - experimentId, sampleId, sampleFileId, file.type, UploadStatus.COMPRESSING, + experimentId, sampleId, sampleFileId, fileType, UploadStatus.COMPRESSING, )); }); @@ -100,7 +102,7 @@ const createAndUploadSampleFile = async (file, experimentId, sampleId, dispatch, : UploadStatus.FILE_READ_ERROR; dispatch(updateSampleFileUpload( - experimentId, sampleId, sampleFileId, file.type, fileErrorStatus, + experimentId, sampleId, sampleFileId, fileType, fileErrorStatus, )); return; } @@ -116,7 +118,7 @@ const createAndUploadSampleFile = async (file, experimentId, sampleId, dispatch, const updateSampleFileUploadProgress = (status, percentProgress = 0) => dispatch( updateSampleFileUpload( - experimentId, sampleId, sampleFileId, file.type, status, percentProgress, + experimentId, sampleId, sampleFileId, fileType, status, percentProgress, ), ); @@ -124,7 +126,7 @@ const createAndUploadSampleFile = async (file, experimentId, sampleId, dispatch, await prepareAndUploadFileToS3(file, uploadUrlParams, 'sample', abortController, updateSampleFileUploadProgress); } catch (e) { dispatch(updateSampleFileUpload( - experimentId, sampleId, sampleFileId, file.type, UploadStatus.UPLOAD_ERROR, + experimentId, sampleId, sampleFileId, fileType, UploadStatus.UPLOAD_ERROR, )); } }; @@ -164,7 +166,7 @@ const processUpload = async (filesList, technology, samples, experimentId, dispa const samplesMap = filesList.reduce((acc, file) => { const { sample: sampleName, name } = getFileSampleAndName(file.fileObject.path.replace(/[\s]{2,}/ig, ' ')); - file.type = fileUploadSpecifications[technology].getCorrespondingType(name); + const fileType = fileUploadSpecifications[technology].getCorrespondingType(name); const sampleUuid = Object.values(samples).filter( (s) => s.name === sampleName @@ -178,7 +180,7 @@ const processUpload = async (filesList, technology, samples, experimentId, dispa uuid: sampleUuid, files: { ...acc[sampleName]?.files, - [file.type]: file, + [fileType]: file, }, }, }; @@ -204,10 +206,11 @@ const processUpload = async (filesList, technology, samples, experimentId, dispa const promises = []; validSamplesList.forEach(([name, sample]) => { - Object.values(sample.files).forEach((file) => { + Object.entries(sample.files).forEach(([type, file]) => { promises.push( async () => await createAndUploadSampleFile( file, + type, experimentId, sampleIdsByName[name], dispatch, @@ -257,6 +260,7 @@ const fileObjectToFileRecord = async (fileObject, technology) => { status: UploadStatus.UPLOADING, progress: 0, }, + errors: error, compressed: verdict === Verdict.VALID_ZIPPED, }; From 9014373482891558a41a4d379a72f21474a5ecc2 Mon Sep 17 00:00:00 2001 From: cosa65 Date: Thu, 7 Dec 2023 10:22:36 -0300 Subject: [PATCH 30/53] Fix Signed-off-by: cosa65 --- src/redux/actions/samples/createSampleFile.js | 2 +- src/redux/actions/samples/createSamples.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/redux/actions/samples/createSampleFile.js b/src/redux/actions/samples/createSampleFile.js index eb027b087b..caeb987f6b 100644 --- a/src/redux/actions/samples/createSampleFile.js +++ b/src/redux/actions/samples/createSampleFile.js @@ -26,7 +26,7 @@ const createSampleFile = ( size: file.size, }; - // Leaving out path, errors, compressed, type + // Leaving out path, errors, compressed // They are used during the upload process, not redux // TODO we should check if they can be removed althogether from the file const fileForRedux = _.pick(file, ['size', 'upload', 'fileObject']); diff --git a/src/redux/actions/samples/createSamples.js b/src/redux/actions/samples/createSamples.js index 0cdd74f717..d34ccbd81a 100644 --- a/src/redux/actions/samples/createSamples.js +++ b/src/redux/actions/samples/createSamples.js @@ -118,7 +118,7 @@ const createSamples = ( { ...acc, [metadataKey]: METADATA_DEFAULT_VALUE }), {}, ) ?? {}, - files: Object.keys(files).reduce(((acc, [fileType]) => ( + files: Object.keys(files).reduce(((acc, fileType) => ( { ...acc, [fileType]: { upload: { status: UploadStatus.UPLOADING } } } )), {}), })); From a37896cbadb34d7075ea9d1bda872166e15b4ddd Mon Sep 17 00:00:00 2001 From: cosa65 Date: Thu, 7 Dec 2023 11:34:36 -0300 Subject: [PATCH 31/53] Fix, store compressed in redux because retry won't work otherwise Signed-off-by: cosa65 --- src/components/data-management/SamplesTableCells.jsx | 3 +-- src/redux/actions/samples/createSampleFile.js | 8 +++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/components/data-management/SamplesTableCells.jsx b/src/components/data-management/SamplesTableCells.jsx index 7f71510da0..dbce4e5313 100644 --- a/src/components/data-management/SamplesTableCells.jsx +++ b/src/components/data-management/SamplesTableCells.jsx @@ -133,12 +133,11 @@ const UploadCell = (props) => { }; const onRetry = () => { - const fileToUpload = _.pick(uploadDetailsModalData, ['size', 'upload', 'fileObject']); const fileType = uploadDetailsModalData.fileCategory; // if retrying an upload we dont need to revalidate the file since it was done before createAndUploadSampleFile( - fileToUpload, + file, fileType, activeExperimentId, sampleUuid, diff --git a/src/redux/actions/samples/createSampleFile.js b/src/redux/actions/samples/createSampleFile.js index caeb987f6b..54c4792a5f 100644 --- a/src/redux/actions/samples/createSampleFile.js +++ b/src/redux/actions/samples/createSampleFile.js @@ -26,10 +26,12 @@ const createSampleFile = ( size: file.size, }; - // Leaving out path, errors, compressed + // Leaving out path, errors // They are used during the upload process, not redux - // TODO we should check if they can be removed althogether from the file - const fileForRedux = _.pick(file, ['size', 'upload', 'fileObject']); + // TODO we should check if they can be separated somehow between + // The ones that are relevant for the api vs + // the ones that are only necessary for retry (fileObject, compressed) + const fileForRedux = _.pick(file, ['size', 'upload', 'fileObject', 'compressed']); dispatch({ type: SAMPLES_FILE_UPDATE, From ed8d5b43e236008bf274208d46095685bd8d97d4 Mon Sep 17 00:00:00 2001 From: cosa65 Date: Thu, 7 Dec 2023 11:35:53 -0300 Subject: [PATCH 32/53] Expand comment Signed-off-by: cosa65 --- src/redux/actions/samples/createSampleFile.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/redux/actions/samples/createSampleFile.js b/src/redux/actions/samples/createSampleFile.js index 54c4792a5f..7c04e68a72 100644 --- a/src/redux/actions/samples/createSampleFile.js +++ b/src/redux/actions/samples/createSampleFile.js @@ -31,6 +31,7 @@ const createSampleFile = ( // TODO we should check if they can be separated somehow between // The ones that are relevant for the api vs // the ones that are only necessary for retry (fileObject, compressed) + // Perhaps into an uploadRetryParams object or something const fileForRedux = _.pick(file, ['size', 'upload', 'fileObject', 'compressed']); dispatch({ From 752500c1c8a540b36c7115fee337086156fc6e83 Mon Sep 17 00:00:00 2001 From: cosa65 Date: Thu, 7 Dec 2023 11:44:19 -0300 Subject: [PATCH 33/53] Rename variable Signed-off-by: cosa65 --- src/components/data-management/SamplesTable.jsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/components/data-management/SamplesTable.jsx b/src/components/data-management/SamplesTable.jsx index c456315091..6f050e0fa3 100644 --- a/src/components/data-management/SamplesTable.jsx +++ b/src/components/data-management/SamplesTable.jsx @@ -228,14 +228,14 @@ const SamplesTable = forwardRef((props, ref) => { }; const generateDataForItem = useCallback((sampleUuid) => { - const sampleFileNames = fileUploadSpecifications[selectedTech]?.requiredFiles + const sampleFileTypes = fileUploadSpecifications[selectedTech]?.requiredFiles .map((requiredFile) => ([requiredFile, { sampleUuid }])); return { key: sampleUuid, name: samples[sampleUuid]?.name || 'UPLOAD ERROR: Please reupload sample', uuid: sampleUuid, - ...Object.fromEntries(sampleFileNames), + ...Object.fromEntries(sampleFileTypes), }; }, [activeExperiment?.sampleIds, selectedTech, samples]); From a66feca42a9b8272469fb5b996541736db3066c4 Mon Sep 17 00:00:00 2001 From: cosa65 Date: Thu, 7 Dec 2023 11:56:34 -0300 Subject: [PATCH 34/53] Remove unnecessary parameters Signed-off-by: cosa65 --- src/components/data-management/SamplesTableCells.jsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/components/data-management/SamplesTableCells.jsx b/src/components/data-management/SamplesTableCells.jsx index dbce4e5313..7c0c8d9ad1 100644 --- a/src/components/data-management/SamplesTableCells.jsx +++ b/src/components/data-management/SamplesTableCells.jsx @@ -128,7 +128,7 @@ const UploadCell = (props) => { }; const onDownload = () => { downloadSampleFile( - activeExperimentId, sampleUuid, uploadDetailsModalData.fileCategory, selectedTech, + activeExperimentId, sampleUuid, uploadDetailsModalData.fileCategory, ); }; From 6dcf508dc2b5b2d26452b9d6491efaae3e72eee7 Mon Sep 17 00:00:00 2001 From: cosa65 Date: Thu, 7 Dec 2023 12:00:51 -0300 Subject: [PATCH 35/53] More explanation in comment Signed-off-by: cosa65 --- src/utils/upload/processUpload.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/utils/upload/processUpload.js b/src/utils/upload/processUpload.js index 121dd06f86..5659377d4d 100644 --- a/src/utils/upload/processUpload.js +++ b/src/utils/upload/processUpload.js @@ -84,7 +84,9 @@ const createAndUploadSampleFile = async ( return; } - // Take the fileName now because after loadAndCompressIfNecessary the name could be lost + // Take the fileName now because after loadAndCompressIfNecessary the name could + // be lost in the compression. If it is compressed fileObject becomes a uInt8Array + // instead of the fileReader metadata object that it is now const fileName = file.fileObject.name; if (!file.compressed) { From cf48f33caeb2fedd1b37b13d993444fc3ef20be7 Mon Sep 17 00:00:00 2001 From: cosa65 Date: Thu, 7 Dec 2023 13:51:39 -0300 Subject: [PATCH 36/53] Fix Signed-off-by: cosa65 --- src/utils/upload/fileUploadSpecifications.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/upload/fileUploadSpecifications.js b/src/utils/upload/fileUploadSpecifications.js index 5d88db532c..1527af5322 100644 --- a/src/utils/upload/fileUploadSpecifications.js +++ b/src/utils/upload/fileUploadSpecifications.js @@ -110,7 +110,7 @@ const fileUploadSpecifications = { name will be used to name the sample in it. You can change this name later in Data Management.`], isNameValid: (fileName) => fileName.toLowerCase().match(/.*matrix.h5(.gz)?$/), - getCorrespondingType: () => 'rhapsody', + getCorrespondingType: () => '10x_h5', }, }; From 35319bdf9eb992dfda6c71f30d768aa80ba10e82 Mon Sep 17 00:00:00 2001 From: cosa65 Date: Thu, 7 Dec 2023 15:08:00 -0300 Subject: [PATCH 37/53] Add trim to handle trailing slashes from files that allow for files drag and drop instead of folder Signed-off-by: cosa65 --- src/components/data-management/FileUploadModal.jsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/components/data-management/FileUploadModal.jsx b/src/components/data-management/FileUploadModal.jsx index 818794d928..9be1e757f1 100644 --- a/src/components/data-management/FileUploadModal.jsx +++ b/src/components/data-management/FileUploadModal.jsx @@ -173,7 +173,7 @@ const FileUploadModal = (props) => { // if the file has a path, trim to just the file and its folder. // otherwise simply use its name fileObject.path - ? Object.values(getFileSampleAndName(fileObject.path)).join('/') + ? _.trim(Object.values(getFileSampleAndName(fileObject.path)).join('/'), '/') : fileObject.name ); From 5afdb41658f72a7ae147d3dc2931a840b1c03f21 Mon Sep 17 00:00:00 2001 From: cosa65 Date: Thu, 7 Dec 2023 15:23:12 -0300 Subject: [PATCH 38/53] Update test Signed-off-by: cosa65 --- .../components/data-management/SamplesTableCells.test.jsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/__test__/components/data-management/SamplesTableCells.test.jsx b/src/__test__/components/data-management/SamplesTableCells.test.jsx index b0df1fc6e0..392a9602d0 100644 --- a/src/__test__/components/data-management/SamplesTableCells.test.jsx +++ b/src/__test__/components/data-management/SamplesTableCells.test.jsx @@ -17,6 +17,7 @@ import mockAPI, { generateDefaultMockAPIResponses, statusResponse } from '__test import { loadSamples, updateSampleFileUpload } from 'redux/actions/samples'; import fake from '__test__/test-utils/constants'; +import sampleFileType from 'utils/sampleFileType'; jest.mock('swr', () => () => ({ data: [ @@ -45,7 +46,7 @@ const sampleFileId = `${fake.SAMPLE_FILE_ID}`; enableFetchMocks(); describe('UploadCell', () => { - const fileCategory = 'features.tsv.gz'; + const fileCategory = sampleFileType.FEATURES_10_X; let storeState = null; From b71d7bb6b4ec96a5ee5fb163c331b2b849a5bafa Mon Sep 17 00:00:00 2001 From: cosa65 Date: Thu, 7 Dec 2023 15:25:57 -0300 Subject: [PATCH 39/53] Update test Signed-off-by: cosa65 --- src/__test__/utils/upload/validateRhapsody.test.js | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/__test__/utils/upload/validateRhapsody.test.js b/src/__test__/utils/upload/validateRhapsody.test.js index 0baaec0f6d..9660a40f46 100644 --- a/src/__test__/utils/upload/validateRhapsody.test.js +++ b/src/__test__/utils/upload/validateRhapsody.test.js @@ -57,12 +57,10 @@ const mockUnzippedSample = { 'expression_data.st', ], files: { - 'expression_data.st': { + rhapsody: { ...sampleFileTemplate, - name: 'expression_data.st', fileObject: mockUnzippedFileObjects['expression_data.st'], size: mockUnzippedFileObjects['expression_data.st'].size, - path: '/sample1/expression_data.st', compressed: false, }, }, @@ -75,8 +73,8 @@ describe('validateRhapsody', () => { it('Throws an error invalid column format', async () => { const mockInvalidColumn = _.cloneDeep(mockUnzippedSample); - mockInvalidColumn.files['expression_data.st'].fileObject = mockUnzippedFileObjects['expression_data_invalid_column.st']; - mockInvalidColumn.files['expression_data.st'].size = mockUnzippedFileObjects['expression_data_invalid_column.st'].size; + mockInvalidColumn.files.rhapsody.fileObject = mockUnzippedFileObjects['expression_data_invalid_column.st']; + mockInvalidColumn.files.rhapsody.size = mockUnzippedFileObjects['expression_data_invalid_column.st'].size; await expect(validateRhapsody(mockInvalidColumn)).rejects.toThrowErrorMatchingSnapshot(); }); From 7438b370f767ad572e0cf84a4798eb495ab66362 Mon Sep 17 00:00:00 2001 From: cosa65 Date: Thu, 7 Dec 2023 15:36:38 -0300 Subject: [PATCH 40/53] Update test Signed-off-by: cosa65 --- src/__test__/utils/upload/validate10x.test.js | 77 +++++-------------- src/redux/reducers/samples/initialState.js | 1 + 2 files changed, 22 insertions(+), 56 deletions(-) diff --git a/src/__test__/utils/upload/validate10x.test.js b/src/__test__/utils/upload/validate10x.test.js index 722e0c4325..5a14773248 100644 --- a/src/__test__/utils/upload/validate10x.test.js +++ b/src/__test__/utils/upload/validate10x.test.js @@ -2,6 +2,7 @@ import validate10x from 'utils/upload/validate10x'; import initialState, { sampleFileTemplate, sampleTemplate } from 'redux/reducers/samples/initialState'; import * as fs from 'fs'; +import sampleFileType from 'utils/sampleFileType'; const _ = require('lodash'); @@ -65,34 +66,23 @@ const mockZippedSample = { ...sampleTemplate, ...initialState, name: 'mockZippedSample', - fileNames: [ - 'features.tsv.gz', - 'barcodes.tsv.gz', - 'matrix.mtx.gz', - ], files: { - 'features.tsv.gz': { + [sampleFileType.FEATURES_10_X]: { ...sampleFileTemplate, - name: 'features.tsv.gz', fileObject: mockZippedFileObjects['features.tsv.gz'], size: mockZippedFileObjects['features.tsv.gz'].size, - path: '/transposed/features.tsv.gz', compressed: true, }, - 'barcodes.tsv.gz': { + [sampleFileType.BARCODES_10_X]: { ...sampleFileTemplate, - name: 'barcodes.tsv.gz', fileObject: mockZippedFileObjects['barcodes.tsv.gz'], size: mockZippedFileObjects['barcodes.tsv.gz'].size, - path: '/transposed/barcodes.tsv.gz', compressed: true, }, - 'matrix.mtx.gz': { + [sampleFileType.MATRIX_10_X]: { ...sampleFileTemplate, - name: 'matrix.mtx.gz', fileObject: mockZippedFileObjects['matrix.mtx.gz'], size: mockZippedFileObjects['matrix.mtx.gz'].size, - path: '/transposed/matrix.mtx.gz', compressed: true, }, }, @@ -102,34 +92,23 @@ const mockUnzippedSample = { ...sampleTemplate, ...initialState, name: 'mockUnzippedSample', - fileNames: [ - 'features.tsv', - 'barcodes.tsv', - 'matrix.mtx', - ], files: { - 'features.tsv': { + [sampleFileType.FEATURES_10_X]: { ...sampleFileTemplate, - name: 'features.tsv', fileObject: mockUnzippedFileObjects['features.tsv'], size: mockUnzippedFileObjects['features.tsv'].size, - path: '/transposed/features.tsv', compressed: false, }, - 'barcodes.tsv': { + [sampleFileType.BARCODES_10_X]: { ...sampleFileTemplate, - name: 'barcodes.tsv', fileObject: mockUnzippedFileObjects['barcodes.tsv'], size: mockUnzippedFileObjects['barcodes.tsv'].size, - path: '/transposed/barcodes.tsv', compressed: false, }, - 'matrix.mtx': { + [sampleFileType.MATRIX_10_X]: { ...sampleFileTemplate, - name: 'matrix.mtx', fileObject: mockUnzippedFileObjects['matrix.mtx'], size: mockUnzippedFileObjects['matrix.mtx'].size, - path: '/transposed/matrix.mtx', compressed: false, }, }, @@ -145,75 +124,61 @@ describe('validate10x', () => { }); it('Throws an error for missing barcodes file', async () => { - const missingFile = 'barcodes.tsv.gz'; - const missingBarcodesFile = _.cloneDeep(mockZippedSample); - missingBarcodesFile.fileNames = missingBarcodesFile.fileNames.filter( - (name) => name !== missingFile, - ); - delete missingBarcodesFile.files[missingFile]; + + delete missingBarcodesFile.files[sampleFileType.BARCODES_10_X]; await expect(validate10x(missingBarcodesFile)).rejects.toThrowErrorMatchingSnapshot(); }); it('Throws an error for missing features file', async () => { - const missingFile = 'features.tsv.gz'; - const missingFeaturesFile = _.cloneDeep(mockZippedSample); - missingFeaturesFile.fileNames = missingFeaturesFile.fileNames.filter( - (name) => name !== missingFile, - ); - delete missingFeaturesFile.files[missingFile]; + delete missingFeaturesFile.files[sampleFileType.FEATURES_10_X]; await expect(validate10x(missingFeaturesFile)).rejects.toThrowErrorMatchingSnapshot(); }); it('Throws an error for missing matrix file', async () => { - const missingFile = 'martix.mtx.gz'; - const missingMatrixFile = _.cloneDeep(mockZippedSample); - missingMatrixFile.fileNames = missingMatrixFile.fileNames.filter( - (name) => name !== missingFile, - ); - delete missingMatrixFile.files[missingFile]; + delete missingMatrixFile.files[sampleFileType.MATRIX_10_X]; }); it('Throws an error matrix with array format', async () => { const mockMatrixArrayFormat = _.cloneDeep(mockZippedSample); - mockMatrixArrayFormat.files['matrix.mtx.gz'].fileObject = mockZippedFileObjects['matrix_array_format.mtx.gz']; - mockMatrixArrayFormat.files['matrix.mtx.gz'].size = mockZippedFileObjects['matrix_array_format.mtx.gz'].size; + mockMatrixArrayFormat.files[sampleFileType.MATRIX_10_X].fileObject = mockZippedFileObjects['matrix_array_format.mtx.gz']; + mockMatrixArrayFormat.files[sampleFileType.MATRIX_10_X].size = mockZippedFileObjects['matrix_array_format.mtx.gz'].size; await expect(validate10x(mockMatrixArrayFormat)).rejects.toThrowErrorMatchingSnapshot(); }); it('Throws an error invalid matrix format', async () => { const mockMatrixNonExistentFormat = _.cloneDeep(mockZippedSample); - mockMatrixNonExistentFormat.files['matrix.mtx.gz'].fileObject = mockZippedFileObjects['matrix_invalid_format.mtx.gz']; - mockMatrixNonExistentFormat.files['matrix.mtx.gz'].size = mockZippedFileObjects['matrix_invalid_format.mtx.gz'].size; + mockMatrixNonExistentFormat.files[sampleFileType.MATRIX_10_X].fileObject = mockZippedFileObjects['matrix_invalid_format.mtx.gz']; + mockMatrixNonExistentFormat.files[sampleFileType.MATRIX_10_X].size = mockZippedFileObjects['matrix_invalid_format.mtx.gz'].size; await expect(validate10x(mockMatrixNonExistentFormat)).rejects.toThrowErrorMatchingSnapshot(); }); it('Throws an error for invalid barcodes file', async () => { const mockInvalidBarcodesFile = _.cloneDeep(mockZippedSample); - mockInvalidBarcodesFile.files['barcodes.tsv.gz'].fileObject = mockZippedFileObjects['invalid_barcodes.tsv.gz']; - mockInvalidBarcodesFile.files['barcodes.tsv.gz'].size = mockZippedFileObjects['invalid_barcodes.tsv.gz'].size; + mockInvalidBarcodesFile.files[sampleFileType.BARCODES_10_X].fileObject = mockZippedFileObjects['invalid_barcodes.tsv.gz']; + mockInvalidBarcodesFile.files[sampleFileType.BARCODES_10_X].size = mockZippedFileObjects['invalid_barcodes.tsv.gz'].size; await expect(validate10x(mockInvalidBarcodesFile)).rejects.toThrowErrorMatchingSnapshot(); }); it('Throws an error for invalid features file', async () => { const mockInvalidFeaturesFile = _.cloneDeep(mockZippedSample); - mockInvalidFeaturesFile.files['features.tsv.gz'].fileObject = mockZippedFileObjects['invalid_features.tsv.gz']; - mockInvalidFeaturesFile.files['features.tsv.gz'].size = mockZippedFileObjects['invalid_features.tsv.gz'].size; + mockInvalidFeaturesFile.files[sampleFileType.FEATURES_10_X].fileObject = mockZippedFileObjects['invalid_features.tsv.gz']; + mockInvalidFeaturesFile.files[sampleFileType.FEATURES_10_X].size = mockZippedFileObjects['invalid_features.tsv.gz'].size; await expect(validate10x(mockInvalidFeaturesFile)).rejects.toThrowErrorMatchingSnapshot(); }); it('Throws an error for transposed matrix file', async () => { const mockTransposedFile = _.cloneDeep(mockZippedSample); - mockTransposedFile.files['matrix.mtx.gz'].fileObject = mockZippedFileObjects['transposed_matrix.mtx.gz']; - mockTransposedFile.files['matrix.mtx.gz'].size = mockZippedFileObjects['transposed_matrix.mtx.gz'].size; + mockTransposedFile.files[sampleFileType.MATRIX_10_X].fileObject = mockZippedFileObjects['transposed_matrix.mtx.gz']; + mockTransposedFile.files[sampleFileType.MATRIX_10_X].size = mockZippedFileObjects['transposed_matrix.mtx.gz'].size; await expect(validate10x(mockTransposedFile)).rejects.toThrowErrorMatchingSnapshot(); }); diff --git a/src/redux/reducers/samples/initialState.js b/src/redux/reducers/samples/initialState.js index 3b493d309b..0655b25356 100644 --- a/src/redux/reducers/samples/initialState.js +++ b/src/redux/reducers/samples/initialState.js @@ -14,6 +14,7 @@ const sampleTemplate = { options: {}, }; +// TODO: Update, this initial state doesn't even match the previously used structure const sampleFileTemplate = { objectKey: '', name: null, From 8099abc714a12b8511884b4d22ceb37c4f6fef61 Mon Sep 17 00:00:00 2001 From: cosa65 Date: Thu, 7 Dec 2023 15:37:46 -0300 Subject: [PATCH 41/53] Update snapshot Signed-off-by: cosa65 --- .../__snapshots__/samplesReducer.test.js.snap | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/src/__test__/redux/reducers/__snapshots__/samplesReducer.test.js.snap b/src/__test__/redux/reducers/__snapshots__/samplesReducer.test.js.snap index a38d819fda..f90a5aebb8 100644 --- a/src/__test__/redux/reducers/__snapshots__/samplesReducer.test.js.snap +++ b/src/__test__/redux/reducers/__snapshots__/samplesReducer.test.js.snap @@ -7,7 +7,6 @@ exports[`samplesReducer Adds a new sample correctly 1`] = ` "createdDate": "2021-01-01T14:48:00.000Z", "error": false, "experimentId": null, - "fileNames": [], "files": {}, "lastModified": "2021-01-01T14:48:00.000Z", "metadata": {}, @@ -27,7 +26,6 @@ exports[`samplesReducer Adds a new sample correctly 1`] = ` "createdDate": "2021-01-02T14:48:00.000Z", "error": false, "experimentId": null, - "fileNames": [], "files": {}, "lastModified": "2021-01-02T14:48:00.000Z", "metadata": {}, @@ -46,7 +44,6 @@ exports[`samplesReducer Adds validating to an experiment 1`] = ` "createdDate": "2021-01-01T14:48:00.000Z", "error": false, "experimentId": null, - "fileNames": [], "files": {}, "lastModified": "2021-01-01T14:48:00.000Z", "metadata": {}, @@ -70,7 +67,6 @@ exports[`samplesReducer Adds validating to an experiment 1`] = ` "createdDate": "2021-01-02T14:48:00.000Z", "error": false, "experimentId": null, - "fileNames": [], "files": {}, "lastModified": "2021-01-02T14:48:00.000Z", "metadata": {}, @@ -91,7 +87,6 @@ exports[`samplesReducer Delete samples correctly 1`] = ` "createdDate": "2021-01-01T14:48:00.000Z", "error": false, "experimentId": null, - "fileNames": [], "files": {}, "lastModified": "2021-01-01T14:48:00.000Z", "metadata": {}, @@ -121,7 +116,6 @@ exports[`samplesReducer Deletes sample metadata correctly 1`] = ` "createdDate": "2021-01-01T14:48:00.000Z", "error": false, "experimentId": null, - "fileNames": [], "files": {}, "lastModified": "2021-01-01T14:48:00.000Z", "metadata": {}, @@ -179,7 +173,6 @@ exports[`samplesReducer Inserts a new sample correctly 1`] = ` "createdDate": "2021-01-01T14:48:00.000Z", "error": false, "experimentId": null, - "fileNames": [], "files": {}, "lastModified": "2021-01-01T14:48:00.000Z", "metadata": {}, @@ -207,7 +200,6 @@ exports[`samplesReducer Inserts sample metadata correctly 1`] = ` "createdDate": "2021-01-01T14:48:00.000Z", "error": false, "experimentId": null, - "fileNames": [], "files": {}, "lastModified": "2021-01-01T14:48:00.000Z", "metadata": { @@ -234,7 +226,6 @@ exports[`samplesReducer Loads samples correctly 1`] = ` "createdDate": "2021-01-01T14:48:00.000Z", "error": false, "experimentId": null, - "fileNames": [], "files": {}, "lastModified": "2021-01-01T14:48:00.000Z", "metadata": {}, @@ -258,7 +249,6 @@ exports[`samplesReducer Loads samples correctly 1`] = ` "createdDate": "2021-01-02T14:48:00.000Z", "error": false, "experimentId": null, - "fileNames": [], "files": {}, "lastModified": "2021-01-02T14:48:00.000Z", "metadata": {}, @@ -277,7 +267,6 @@ exports[`samplesReducer Removes validating from an experiment 1`] = ` "createdDate": "2021-01-01T14:48:00.000Z", "error": false, "experimentId": null, - "fileNames": [], "files": {}, "lastModified": "2021-01-01T14:48:00.000Z", "metadata": {}, @@ -298,7 +287,6 @@ exports[`samplesReducer Removes validating from an experiment 1`] = ` "createdDate": "2021-01-02T14:48:00.000Z", "error": false, "experimentId": null, - "fileNames": [], "files": {}, "lastModified": "2021-01-02T14:48:00.000Z", "metadata": {}, @@ -319,7 +307,6 @@ exports[`samplesReducer Sets up saved state correctly 1`] = ` "createdDate": "2021-01-01T14:48:00.000Z", "error": false, "experimentId": null, - "fileNames": [], "files": {}, "lastModified": "2021-01-01T14:48:00.000Z", "metadata": {}, @@ -343,7 +330,6 @@ exports[`samplesReducer Sets up saving state correctly 1`] = ` "createdDate": "2021-01-01T14:48:00.000Z", "error": false, "experimentId": null, - "fileNames": [], "files": {}, "lastModified": "2021-01-01T14:48:00.000Z", "metadata": {}, @@ -367,7 +353,6 @@ exports[`samplesReducer Stores error state correctly 1`] = ` "createdDate": "2021-01-01T14:48:00.000Z", "error": false, "experimentId": null, - "fileNames": [], "files": {}, "lastModified": "2021-01-01T14:48:00.000Z", "metadata": {}, @@ -391,7 +376,6 @@ exports[`samplesReducer Updates a sample correctly 1`] = ` "createdDate": "2021-01-01T14:48:00.000Z", "error": false, "experimentId": null, - "fileNames": [], "files": {}, "lastModified": "2021-01-01T14:48:00.000Z", "metadata": {}, @@ -416,7 +400,6 @@ exports[`samplesReducer Updates options correctly 1`] = ` "createdDate": "2021-01-01T14:48:00.000Z", "error": false, "experimentId": null, - "fileNames": [], "files": {}, "lastModified": "2021-01-01T14:48:00.000Z", "metadata": {}, @@ -438,7 +421,6 @@ exports[`samplesReducer Updates options correctly 1`] = ` "createdDate": "2021-01-02T14:48:00.000Z", "error": false, "experimentId": null, - "fileNames": [], "files": {}, "lastModified": "2021-01-02T14:48:00.000Z", "metadata": {}, From 0e7f64deba63591fcad9c20d0560919cae910ca1 Mon Sep 17 00:00:00 2001 From: cosa65 Date: Thu, 7 Dec 2023 15:38:29 -0300 Subject: [PATCH 42/53] Update test Signed-off-by: cosa65 --- src/__test__/redux/reducers/samplesReducer.test.js | 1 - 1 file changed, 1 deletion(-) diff --git a/src/__test__/redux/reducers/samplesReducer.test.js b/src/__test__/redux/reducers/samplesReducer.test.js index 780b919e6f..8a2f40eb4d 100644 --- a/src/__test__/redux/reducers/samplesReducer.test.js +++ b/src/__test__/redux/reducers/samplesReducer.test.js @@ -55,7 +55,6 @@ describe('samplesReducer', () => { const mockFile = { ...sampleFileTemplate, - name: fileName, }; it('Reduces identical state on unknown action', () => expect( From 715fc04cc458abb632bf3cc59df21dba84af2a49 Mon Sep 17 00:00:00 2001 From: cosa65 Date: Thu, 7 Dec 2023 15:45:34 -0300 Subject: [PATCH 43/53] Update initialState Signed-off-by: cosa65 --- src/redux/reducers/samples/initialState.js | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/redux/reducers/samples/initialState.js b/src/redux/reducers/samples/initialState.js index 0655b25356..e85bc464c8 100644 --- a/src/redux/reducers/samples/initialState.js +++ b/src/redux/reducers/samples/initialState.js @@ -17,16 +17,11 @@ const sampleTemplate = { // TODO: Update, this initial state doesn't even match the previously used structure const sampleFileTemplate = { objectKey: '', - name: null, size: 0, - mime: '', - path: '', success: false, error: false, - lastModified: '', upload: { status: null, - amplifyPromise: null, }, }; From 281ccf1a4141f2d68cb7a6a226150aefbb2ece15 Mon Sep 17 00:00:00 2001 From: cosa65 Date: Thu, 7 Dec 2023 15:45:47 -0300 Subject: [PATCH 44/53] Update test Signed-off-by: cosa65 --- .../__snapshots__/samplesReducer.test.js.snap | 10 +--------- src/__test__/redux/reducers/samplesReducer.test.js | 12 ++++-------- 2 files changed, 5 insertions(+), 17 deletions(-) diff --git a/src/__test__/redux/reducers/__snapshots__/samplesReducer.test.js.snap b/src/__test__/redux/reducers/__snapshots__/samplesReducer.test.js.snap index f90a5aebb8..d5517b9e74 100644 --- a/src/__test__/redux/reducers/__snapshots__/samplesReducer.test.js.snap +++ b/src/__test__/redux/reducers/__snapshots__/samplesReducer.test.js.snap @@ -441,21 +441,13 @@ exports[`samplesReducer Updates sample files correctly 1`] = ` "createdDate": "2021-01-01T14:48:00.000Z", "error": false, "experimentId": null, - "fileNames": [ - "features.tsv", - ], "files": { - "features.tsv": { + "features10x": { "error": false, - "lastModified": "newLastModified", - "mime": "", - "name": "features.tsv", "objectKey": "", - "path": "", "size": 0, "success": false, "upload": { - "amplifyPromise": null, "status": null, }, }, diff --git a/src/__test__/redux/reducers/samplesReducer.test.js b/src/__test__/redux/reducers/samplesReducer.test.js index 8a2f40eb4d..cdeccf286c 100644 --- a/src/__test__/redux/reducers/samplesReducer.test.js +++ b/src/__test__/redux/reducers/samplesReducer.test.js @@ -16,11 +16,11 @@ import { SAMPLES_VALIDATING_UPDATED, } from 'redux/actionTypes/samples'; import { EXPERIMENTS_METADATA_RENAME } from 'redux/actionTypes/experiments'; +import sampleFileType from 'utils/sampleFileType'; describe('samplesReducer', () => { const mockUuid1 = 'asd123'; const mockUuid2 = 'qwe234'; - const fileName = 'features.tsv'; const sample1 = { ...sampleTemplate, @@ -107,17 +107,13 @@ describe('samplesReducer', () => { type: SAMPLES_FILE_UPDATE, payload: { sampleUuid: mockUuid1, - fileName, + sampleFileType: sampleFileType.FEATURES_10_X, fileDiff: mockFile, lastModified: 'newLastModified', }, }); - expect(newState[sample1.uuid].fileNames).toEqual([fileName]); - expect(newState[sample1.uuid].files[fileName]).toEqual({ - ...mockFile, - lastModified: 'newLastModified', - }); + expect(newState[sample1.uuid].files[sampleFileType.FEATURES_10_X]).toEqual(mockFile); expect(newState).toMatchSnapshot(); }); @@ -403,7 +399,7 @@ describe('samplesReducer', () => { type: SAMPLES_FILE_UPDATE, payload: { sampleUuid: mockUuid1, - fileName, + sampleFileType: sampleFileType.FEATURES_10_X, fileDiff: mockFile, lastModified: 'newLastModified', }, From beea7ceb54029dcb74a1b03c17298b1c98dc4232 Mon Sep 17 00:00:00 2001 From: cosa65 Date: Thu, 7 Dec 2023 15:47:26 -0300 Subject: [PATCH 45/53] Update test Signed-off-by: cosa65 --- .../utils/data-management/downloadSingleFile.test.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/__test__/utils/data-management/downloadSingleFile.test.js b/src/__test__/utils/data-management/downloadSingleFile.test.js index 1c9d36d265..f3abcb513d 100644 --- a/src/__test__/utils/data-management/downloadSingleFile.test.js +++ b/src/__test__/utils/data-management/downloadSingleFile.test.js @@ -5,6 +5,7 @@ import downloadFromUrl from 'utils/downloadFromUrl'; import { sampleTech } from 'utils/constants'; import fake from '__test__/test-utils/constants'; +import sampleFileType from 'utils/sampleFileType'; jest.mock('utils/downloadFromUrl'); @@ -23,9 +24,7 @@ describe('downloadFromUrl', () => { fetchMock.mockResponse(JSON.stringify(mockSignedUrl)); - const fileName = 'features.tsv.gz'; - - await downloadSingleFile(fake.EXPERIMENT_ID, fake.SAMPLE_ID, fileName, sampleTech['10X']); + await downloadSingleFile(fake.EXPERIMENT_ID, fake.SAMPLE_ID, sampleFileType.FEATURES_10_X, sampleTech['10X']); expect(downloadFromUrl).toHaveBeenCalledWith(mockSignedUrl); expect(fetchMock.mock.calls).toMatchSnapshot(); From 2fd2f9f094be8d3fe59160e02b4033d694fe8ecc Mon Sep 17 00:00:00 2001 From: cosa65 Date: Thu, 7 Dec 2023 15:55:00 -0300 Subject: [PATCH 46/53] Update test Signed-off-by: cosa65 --- .../UploadDetailsModal.test.jsx | 37 +++++++++---------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/src/__test__/components/data-management/UploadDetailsModal.test.jsx b/src/__test__/components/data-management/UploadDetailsModal.test.jsx index d8b5722c40..667e33afab 100644 --- a/src/__test__/components/data-management/UploadDetailsModal.test.jsx +++ b/src/__test__/components/data-management/UploadDetailsModal.test.jsx @@ -5,7 +5,7 @@ import { import '@testing-library/jest-dom'; import userEvent from '@testing-library/user-event'; import UploadDetailsModal from 'components/data-management/UploadDetailsModal'; -import UploadStatus, { messageForStatus } from 'utils/upload/UploadStatus'; +import UploadStatus from 'utils/upload/UploadStatus'; const mockOnDelete = jest.fn(); const mockOnCancel = jest.fn(); @@ -15,7 +15,6 @@ const mockOnRetry = jest.fn(); const defaultProps = { visible: true, data: { - name: 'example.txt', size: 1024, lastModified: new Date().toISOString(), upload: { @@ -41,9 +40,9 @@ describe('UploadDetailsModal', () => { it('displays the modal with the correct title when uploaded', () => { renderUploadDetailsModal({ - file: { - ...defaultProps.file, - upload: { ...defaultProps.file.upload, status: UploadStatus.UPLOADED }, + data: { + ...defaultProps.data, + upload: { ...defaultProps.data.upload, status: UploadStatus.UPLOADED }, }, }); @@ -52,8 +51,8 @@ describe('UploadDetailsModal', () => { it('displays the modal with the correct title when file not found', () => { renderUploadDetailsModal({ - file: { - ...defaultProps.file, + data: { + ...defaultProps.data, upload: { status: UploadStatus.FILE_NOT_FOUND }, }, }); @@ -62,9 +61,9 @@ describe('UploadDetailsModal', () => { it('calls onRetry when the retry button is clicked', async () => { renderUploadDetailsModal({ - file: { - ...defaultProps.file, - upload: { ...defaultProps.file.upload, status: UploadStatus.ERROR }, + data: { + ...defaultProps.data, + upload: { ...defaultProps.data.upload, status: UploadStatus.ERROR }, fileObject: [1, 2, 3, 4, 5, 6], }, }); @@ -79,9 +78,9 @@ describe('UploadDetailsModal', () => { it('calls onDownload when the download button is clicked', () => { renderUploadDetailsModal({ - file: { - ...defaultProps.file, - upload: { ...defaultProps.file.upload, status: UploadStatus.UPLOADED }, + data: { + ...defaultProps.data, + upload: { ...defaultProps.data.upload, status: UploadStatus.UPLOADED }, }, }); @@ -94,9 +93,9 @@ describe('UploadDetailsModal', () => { it('renders error message when there is an upload error', () => { const status = UploadStatus.UPLOAD_ERROR; renderUploadDetailsModal({ - file: { - ...defaultProps.file, - upload: { ...defaultProps.file.upload, status }, + data: { + ...defaultProps.data, + upload: { ...defaultProps.data.upload, status }, }, }); @@ -106,9 +105,9 @@ describe('UploadDetailsModal', () => { it('Deleting the file calls onDelete', () => { const status = UploadStatus.UPLOADED; renderUploadDetailsModal({ - file: { - ...defaultProps.file, - upload: { ...defaultProps.file.upload, status }, + data: { + ...defaultProps.data, + upload: { ...defaultProps.data.upload, status }, }, }); From 6366d3cc6bfa1b356bc3f032e12c84c682e2ae6f Mon Sep 17 00:00:00 2001 From: cosa65 Date: Thu, 7 Dec 2023 15:56:28 -0300 Subject: [PATCH 47/53] Update snapshot Signed-off-by: cosa65 --- .../__snapshots__/loadSamples.test.js.snap | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/src/__test__/redux/actions/samples/__snapshots__/loadSamples.test.js.snap b/src/__test__/redux/actions/samples/__snapshots__/loadSamples.test.js.snap index a62ad8d8d4..b8c76b0951 100644 --- a/src/__test__/redux/actions/samples/__snapshots__/loadSamples.test.js.snap +++ b/src/__test__/redux/actions/samples/__snapshots__/loadSamples.test.js.snap @@ -15,35 +15,24 @@ exports[`loadSample action Works correctly 1`] = ` "e03ef6ea-5014-4e57-aecd-59964ac9172c": { "createdDate": "2021-12-07 17:36:27.773+00", "experimentId": "1234", - "fileNames": [ - "matrix.mtx.gz", - "barcodes.tsv.gz", - "features.tsv.gz", - ], "files": { - "barcodes.tsv.gz": { - "name": "barcodes.tsv.gz", + "barcodes10x": { "size": undefined, "upload": { "status": "uploaded", }, - "valid": true, }, - "features.tsv.gz": { - "name": "features.tsv.gz", + "features10x": { "size": undefined, "upload": { "status": "uploaded", }, - "valid": true, }, - "matrix.mtx.gz": { - "name": "matrix.mtx.gz", + "matrix10x": { "size": undefined, "upload": { "status": "uploaded", }, - "valid": true, }, }, "lastModified": "2021-12-07 17:38:42.036+00", From 3599210c82361c76471ddbfe8fb28ca6f9c0e6fe Mon Sep 17 00:00:00 2001 From: cosa65 Date: Tue, 12 Dec 2023 09:00:30 -0300 Subject: [PATCH 48/53] data is always expected to be defined, remove Signed-off-by: cosa65 --- src/components/data-management/UploadDetailsModal.jsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/components/data-management/UploadDetailsModal.jsx b/src/components/data-management/UploadDetailsModal.jsx index 3f609fb289..4eb2c943a2 100644 --- a/src/components/data-management/UploadDetailsModal.jsx +++ b/src/components/data-management/UploadDetailsModal.jsx @@ -17,7 +17,7 @@ const UploadDetailsModal = (props) => { const { upload, size, lastModified, fileObject = undefined, - } = data ?? {}; + } = data; const { progress, status } = upload ?? false; From 4aab83c26461b735d63c3f7426744caa8bfec3da Mon Sep 17 00:00:00 2001 From: cosa65 Date: Tue, 12 Dec 2023 09:19:13 -0300 Subject: [PATCH 49/53] Fix, don't use booleans to indicate null Signed-off-by: cosa65 --- src/components/data-management/SamplesTableCells.jsx | 2 +- src/components/data-management/UploadDetailsModal.jsx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/components/data-management/SamplesTableCells.jsx b/src/components/data-management/SamplesTableCells.jsx index 7c0c8d9ad1..7b94492817 100644 --- a/src/components/data-management/SamplesTableCells.jsx +++ b/src/components/data-management/SamplesTableCells.jsx @@ -40,7 +40,7 @@ const UploadCell = (props) => { const { activeExperimentId } = useSelector((state) => state.experiments.meta); const [uploadDetailsModalVisible, setUploadDetailsModalVisible] = useState(false); - const [uploadDetailsModalData, setUploadDetailsModalData] = useState(false); + const [uploadDetailsModalData, setUploadDetailsModalData] = useState(null); useEffect(() => { setUploadDetailsModalData(file); diff --git a/src/components/data-management/UploadDetailsModal.jsx b/src/components/data-management/UploadDetailsModal.jsx index 4eb2c943a2..80fd0c5a96 100644 --- a/src/components/data-management/UploadDetailsModal.jsx +++ b/src/components/data-management/UploadDetailsModal.jsx @@ -19,7 +19,7 @@ const UploadDetailsModal = (props) => { upload, size, lastModified, fileObject = undefined, } = data; - const { progress, status } = upload ?? false; + const { progress, status } = upload; const isSuccessModal = status === UploadStatus.UPLOADED; const isNotUploadedModal = status === UploadStatus.FILE_NOT_FOUND; From d5298fcd4db1f06ea6b0ba53f863358335ddea21 Mon Sep 17 00:00:00 2001 From: cosa65 Date: Tue, 12 Dec 2023 09:40:37 -0300 Subject: [PATCH 50/53] Minor readibility improvements, remove unnecessary optional operators Signed-off-by: cosa65 --- .../data-management/SamplesTable.jsx | 79 ++++++++++--------- 1 file changed, 42 insertions(+), 37 deletions(-) diff --git a/src/components/data-management/SamplesTable.jsx b/src/components/data-management/SamplesTable.jsx index 6f050e0fa3..b6775f8da2 100644 --- a/src/components/data-management/SamplesTable.jsx +++ b/src/components/data-management/SamplesTable.jsx @@ -71,47 +71,51 @@ const SamplesTable = forwardRef((props, ref) => { const [samplesLoaded, setSamplesLoaded] = useState(false); - const initialTableColumns = useMemo(() => ([ - { - fixed: 'left', - index: 0, - key: 'sort', - dataIndex: 'sort', - width: 50, - render: () => , - }, - { - className: `${integrationTestConstants.classes.SAMPLE_CELL}`, - index: 1, - key: 'sample', - title: selectedTech === sampleTech.SEURAT ? 'File' : 'Sample', - dataIndex: 'name', - fixed: 'left', - render: (text, record, indx) => ( - - ), - }, - ...fileUploadSpecifications[selectedTech]?.requiredFiles?.map((requiredFile, indx) => ({ - index: 2 + indx, - title:
{fileTypeToDisplay[requiredFile]}
, - key: requiredFile, - dataIndex: requiredFile, - width: 170, - onCell: () => ({ style: { margin: '0px', padding: '0px' } }), - render: (tableCellData) => tableCellData && ( - - ), - })) || [], + const initialTableColumns = useMemo(() => { + if (_.isNil(selectedTech)) return []; - ]), [selectedTech]); + return ([ + { + fixed: 'left', + index: 0, + key: 'sort', + dataIndex: 'sort', + width: 50, + render: () => , + }, + { + className: `${integrationTestConstants.classes.SAMPLE_CELL}`, + index: 1, + key: 'sample', + title: selectedTech === sampleTech.SEURAT ? 'File' : 'Sample', + dataIndex: 'name', + fixed: 'left', + render: (text, record, indx) => ( + + ), + }, + ...fileUploadSpecifications[selectedTech].requiredFiles.map((requiredFile, indx) => ({ + index: 2 + indx, + title:
{fileTypeToDisplay[requiredFile]}
, + key: requiredFile, + dataIndex: requiredFile, + width: 170, + onCell: () => ({ style: { margin: '0px', padding: '0px' } }), + render: (tableCellData) => tableCellData && ( + + ), + })), + + ]); + }, [selectedTech]); const [tableColumns, setTableColumns] = useState(initialTableColumns); useEffect(() => { - if (activeExperiment?.sampleIds.length > 0 && samplesLoaded) { + if (activeExperiment?.sampleIds.length > 0 && !samplesLoading) { // if there are samples - build the table columns const sanitizedSampleNames = new Set( activeExperiment.sampleIds.map((id) => samples[id]?.name.trim()), @@ -121,9 +125,10 @@ const SamplesTable = forwardRef((props, ref) => { const metadataColumns = activeExperiment.metadataKeys.map( (metadataKey) => createInitializedMetadataColumn(metadataKeyToName(metadataKey)), ) || []; + setTableColumns([...initialTableColumns, ...metadataColumns]); } - }, [samples, activeExperiment?.sampleIds, samplesLoaded]); + }, [samples, activeExperiment?.sampleIds, samplesLoading]); useConditionalEffect(() => { setSamplesLoaded(false); From 1eeccaa57ce564b0cd739128f10ecdbdcaede51f Mon Sep 17 00:00:00 2001 From: cosa65 Date: Tue, 12 Dec 2023 09:53:56 -0300 Subject: [PATCH 51/53] Remove case fileObject.path not set check, it seems to no longer be a possibility Signed-off-by: cosa65 --- src/components/data-management/FileUploadModal.jsx | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/src/components/data-management/FileUploadModal.jsx b/src/components/data-management/FileUploadModal.jsx index 9be1e757f1..cc3180a4a8 100644 --- a/src/components/data-management/FileUploadModal.jsx +++ b/src/components/data-management/FileUploadModal.jsx @@ -169,13 +169,7 @@ const FileUploadModal = (props) => { ); - const getFilePathToDisplay = (fileObject) => ( - // if the file has a path, trim to just the file and its folder. - // otherwise simply use its name - fileObject.path - ? _.trim(Object.values(getFileSampleAndName(fileObject.path)).join('/'), '/') - : fileObject.name - ); + const getFilePathToDisplay = (fileObject) => _.trim(Object.values(getFileSampleAndName(fileObject.path)).join('/'), '/'); return ( Date: Tue, 12 Dec 2023 10:12:32 -0300 Subject: [PATCH 52/53] Update test Signed-off-by: cosa65 --- src/__test__/utils/upload/process10XUpload.test.js | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/__test__/utils/upload/process10XUpload.test.js b/src/__test__/utils/upload/process10XUpload.test.js index 7fa6b982b7..9698d3b98a 100644 --- a/src/__test__/utils/upload/process10XUpload.test.js +++ b/src/__test__/utils/upload/process10XUpload.test.js @@ -25,28 +25,28 @@ import loadAndCompressIfNecessary from 'utils/upload/loadAndCompressIfNecessary' enableFetchMocks(); const getValidFiles = (cellrangerVersion, compressed = true) => { - const filename = cellrangerVersion === 'v2' ? 'genes.tsv.gz' : 'features.tsv.gz'; + const featuresFilename = cellrangerVersion === 'v2' ? 'genes.tsv.gz' : 'features.tsv.gz'; let fileList = [ { - name: `WT13/${filename}`, - fileObject: mockFile(filename, '/'), + name: `${featuresFilename}`, + fileObject: mockFile(featuresFilename, 'WT13'), upload: { status: UploadStatus.UPLOADING }, errors: '', compressed, valid: true, }, { - name: 'WT13/barcodes.tsv.gz', - fileObject: mockFile('barcodes.tsv.gz', '/'), + name: 'barcodes.tsv.gz', + fileObject: mockFile('barcodes.tsv.gz', 'WT13'), upload: { status: UploadStatus.UPLOADING }, errors: '', compressed, valid: true, }, { - name: 'WT13/matrix.mtx.gz', - fileObject: mockFile('matrix.mtx.gz', '/'), + name: 'matrix.mtx.gz', + fileObject: mockFile('matrix.mtx.gz', 'WT13'), upload: { status: UploadStatus.UPLOADING }, errors: '', compressed, From 20a052c72c8244f6bb08500467aa579833437905 Mon Sep 17 00:00:00 2001 From: cosa65 Date: Tue, 12 Dec 2023 11:30:31 -0300 Subject: [PATCH 53/53] Update snapshot Signed-off-by: cosa65 --- .../switchExperiment.test.js.snap | 102 ++++-------------- 1 file changed, 18 insertions(+), 84 deletions(-) diff --git a/src/__test__/redux/actions/experiments/__snapshots__/switchExperiment.test.js.snap b/src/__test__/redux/actions/experiments/__snapshots__/switchExperiment.test.js.snap index 149ad37183..cb702417eb 100644 --- a/src/__test__/redux/actions/experiments/__snapshots__/switchExperiment.test.js.snap +++ b/src/__test__/redux/actions/experiments/__snapshots__/switchExperiment.test.js.snap @@ -328,35 +328,24 @@ exports[`switch experiment Then updates the experiment info 1`] = ` "test9188-d682-test-mock-cb6d644cmock-0": { "createdDate": "2021-12-07 17:36:27.773+00", "experimentId": "testae48e318dab9a1bd0bexperiment-0", - "fileNames": [ - "matrix.mtx.gz", - "barcodes.tsv.gz", - "features.tsv.gz", - ], "files": { - "barcodes.tsv.gz": { - "name": "barcodes.tsv.gz", + "barcodes10x": { "size": 100, "upload": { "status": "uploaded", }, - "valid": true, }, - "features.tsv.gz": { - "name": "features.tsv.gz", + "features10x": { "size": 100, "upload": { "status": "uploaded", }, - "valid": true, }, - "matrix.mtx.gz": { - "name": "matrix.mtx.gz", + "matrix10x": { "size": 1000, "upload": { "status": "uploaded", }, - "valid": true, }, }, "lastModified": "2021-12-07 17:38:42.036+00", @@ -372,35 +361,24 @@ exports[`switch experiment Then updates the experiment info 1`] = ` "test9188-d682-test-mock-cb6d644cmock-1": { "createdDate": "2021-12-07 17:36:27.773+00", "experimentId": "testae48e318dab9a1bd0bexperiment-0", - "fileNames": [ - "matrix.mtx.gz", - "barcodes.tsv.gz", - "features.tsv.gz", - ], "files": { - "barcodes.tsv.gz": { - "name": "barcodes.tsv.gz", + "barcodes10x": { "size": 100, "upload": { "status": "uploaded", }, - "valid": true, }, - "features.tsv.gz": { - "name": "features.tsv.gz", + "features10x": { "size": 100, "upload": { "status": "uploaded", }, - "valid": true, }, - "matrix.mtx.gz": { - "name": "matrix.mtx.gz", + "matrix10x": { "size": 1000, "upload": { "status": "uploaded", }, - "valid": true, }, }, "lastModified": "2021-12-07 17:38:42.036+00", @@ -416,35 +394,24 @@ exports[`switch experiment Then updates the experiment info 1`] = ` "test9188-d682-test-mock-cb6d644cmock-2": { "createdDate": "2021-12-07 17:36:27.773+00", "experimentId": "testae48e318dab9a1bd0bexperiment-0", - "fileNames": [ - "matrix.mtx.gz", - "barcodes.tsv.gz", - "features.tsv.gz", - ], "files": { - "barcodes.tsv.gz": { - "name": "barcodes.tsv.gz", + "barcodes10x": { "size": 100, "upload": { "status": "uploaded", }, - "valid": true, }, - "features.tsv.gz": { - "name": "features.tsv.gz", + "features10x": { "size": 100, "upload": { "status": "uploaded", }, - "valid": true, }, - "matrix.mtx.gz": { - "name": "matrix.mtx.gz", + "matrix10x": { "size": 1000, "upload": { "status": "uploaded", }, - "valid": true, }, }, "lastModified": "2021-12-07 17:38:42.036+00", @@ -792,35 +759,24 @@ exports[`switch experiment switches the experiment to its initial values 1`] = "test9188-d682-test-mock-cb6d644cmock-0": { "createdDate": "2021-12-07 17:36:27.773+00", "experimentId": "testae48e318dab9a1bd0bexperiment-0", - "fileNames": [ - "matrix.mtx.gz", - "barcodes.tsv.gz", - "features.tsv.gz", - ], "files": { - "barcodes.tsv.gz": { - "name": "barcodes.tsv.gz", + "barcodes10x": { "size": 100, "upload": { "status": "uploaded", }, - "valid": true, }, - "features.tsv.gz": { - "name": "features.tsv.gz", + "features10x": { "size": 100, "upload": { "status": "uploaded", }, - "valid": true, }, - "matrix.mtx.gz": { - "name": "matrix.mtx.gz", + "matrix10x": { "size": 1000, "upload": { "status": "uploaded", }, - "valid": true, }, }, "lastModified": "2021-12-07 17:38:42.036+00", @@ -836,35 +792,24 @@ exports[`switch experiment switches the experiment to its initial values 1`] = "test9188-d682-test-mock-cb6d644cmock-1": { "createdDate": "2021-12-07 17:36:27.773+00", "experimentId": "testae48e318dab9a1bd0bexperiment-0", - "fileNames": [ - "matrix.mtx.gz", - "barcodes.tsv.gz", - "features.tsv.gz", - ], "files": { - "barcodes.tsv.gz": { - "name": "barcodes.tsv.gz", + "barcodes10x": { "size": 100, "upload": { "status": "uploaded", }, - "valid": true, }, - "features.tsv.gz": { - "name": "features.tsv.gz", + "features10x": { "size": 100, "upload": { "status": "uploaded", }, - "valid": true, }, - "matrix.mtx.gz": { - "name": "matrix.mtx.gz", + "matrix10x": { "size": 1000, "upload": { "status": "uploaded", }, - "valid": true, }, }, "lastModified": "2021-12-07 17:38:42.036+00", @@ -880,35 +825,24 @@ exports[`switch experiment switches the experiment to its initial values 1`] = "test9188-d682-test-mock-cb6d644cmock-2": { "createdDate": "2021-12-07 17:36:27.773+00", "experimentId": "testae48e318dab9a1bd0bexperiment-0", - "fileNames": [ - "matrix.mtx.gz", - "barcodes.tsv.gz", - "features.tsv.gz", - ], "files": { - "barcodes.tsv.gz": { - "name": "barcodes.tsv.gz", + "barcodes10x": { "size": 100, "upload": { "status": "uploaded", }, - "valid": true, }, - "features.tsv.gz": { - "name": "features.tsv.gz", + "features10x": { "size": 100, "upload": { "status": "uploaded", }, - "valid": true, }, - "matrix.mtx.gz": { - "name": "matrix.mtx.gz", + "matrix10x": { "size": 1000, "upload": { "status": "uploaded", }, - "valid": true, }, }, "lastModified": "2021-12-07 17:38:42.036+00",