From eab8a94c7a6ac2bd8251033fb9d3cac888f452c4 Mon Sep 17 00:00:00 2001 From: Anugerah Erlaut Date: Fri, 15 Jul 2022 06:58:38 +0100 Subject: [PATCH 01/15] add sample check --- .../data-management/SamplesTable.jsx | 6 +- .../data-management/SamplesTableCells.jsx | 27 ++-- src/redux/actions/samples/createSample.js | 4 + src/redux/reducers/samples/initialState.js | 2 + src/utils/upload/processUpload.js | 16 +- src/utils/upload/sampleInspector.js | 145 ++++++++++++++++++ 6 files changed, 189 insertions(+), 11 deletions(-) create mode 100644 src/utils/upload/sampleInspector.js diff --git a/src/components/data-management/SamplesTable.jsx b/src/components/data-management/SamplesTable.jsx index 2abf47990e..3bcbd7f2d1 100644 --- a/src/components/data-management/SamplesTable.jsx +++ b/src/components/data-management/SamplesTable.jsx @@ -230,10 +230,14 @@ const SamplesTable = forwardRef((props, ref) => { const genesData = { sampleUuid, file: genesFile }; const matrixData = { sampleUuid, file: matrixFile }; + const { name, valid, validationMessage } = samples[sampleUuid] ?? {}; + return { key: idx, - name: samples[sampleUuid]?.name || 'UPLOAD ERROR: Please reupload sample', + name: name || 'UPLOAD ERROR: Please reupload sample', uuid: sampleUuid, + valid: valid || false, + validationMessage: validationMessage || '', barcodes: barcodesData, genes: genesData, matrix: matrixData, diff --git a/src/components/data-management/SamplesTableCells.jsx b/src/components/data-management/SamplesTableCells.jsx index ab7d369ced..513a0027ef 100644 --- a/src/components/data-management/SamplesTableCells.jsx +++ b/src/components/data-management/SamplesTableCells.jsx @@ -3,7 +3,7 @@ import { Space, Typography, Progress, Tooltip, Button, } from 'antd'; import { - UploadOutlined, + UploadOutlined, WarningFilled, } from '@ant-design/icons'; import { useDispatch } from 'react-redux'; import PropTypes from 'prop-types'; @@ -167,14 +167,23 @@ const SampleNameCell = (props) => { const { text, record, idx } = cellInfo; const dispatch = useDispatch(); return ( - - dispatch(updateSample(record.uuid, { name }))} - onDelete={() => dispatch(deleteSamples([record.uuid]))} - /> - + + + dispatch(updateSample(record.uuid, { name }))} + onDelete={() => dispatch(deleteSamples([record.uuid]))} + /> + + { + !record.valid ? ( + + + + ) : '' + } + ); }; SampleNameCell.propTypes = { diff --git a/src/redux/actions/samples/createSample.js b/src/redux/actions/samples/createSample.js index 0b6dbcd548..c8b3beef2f 100644 --- a/src/redux/actions/samples/createSample.js +++ b/src/redux/actions/samples/createSample.js @@ -18,6 +18,8 @@ const createSample = ( experimentId, name, type, + valid, + validationMessage, filesToUpload, ) => async (dispatch, getState) => { const experiment = getState().experiments[experimentId]; @@ -38,6 +40,8 @@ const createSample = ( type, experimentId, uuid: newSampleUuid, + valid, + validationMessage, createdDate, lastModified: createdDate, metadata: experiment?.metadataKeys diff --git a/src/redux/reducers/samples/initialState.js b/src/redux/reducers/samples/initialState.js index 2b83ad2c68..b1c970db0b 100644 --- a/src/redux/reducers/samples/initialState.js +++ b/src/redux/reducers/samples/initialState.js @@ -3,6 +3,8 @@ const sampleTemplate = { experimentId: null, uuid: null, type: null, + valid: null, + validationMessage: null, createdDate: null, lastModified: null, complete: false, diff --git a/src/utils/upload/processUpload.js b/src/utils/upload/processUpload.js index 7a80ac82e7..9ba4b276f4 100644 --- a/src/utils/upload/processUpload.js +++ b/src/utils/upload/processUpload.js @@ -4,6 +4,7 @@ import _ from 'lodash'; import axios from 'axios'; import { createSample, createSampleFile, updateSampleFileUpload } from 'redux/actions/samples'; +import { inspectSample, verdictText } from 'utils/upload/sampleInspector'; import UploadStatus from 'utils/upload/UploadStatus'; import loadAndCompressIfNecessary from 'utils/upload/loadAndCompressIfNecessary'; @@ -129,12 +130,25 @@ const processUpload = async (filesList, sampleType, samples, experimentId, dispa }, {}); Object.entries(samplesMap).forEach(async ([name, sample]) => { + console.log('*** inspecting sample', name); + + // Validate sample + const { valid, verdict } = await inspectSample(sample); + const validationMessage = verdict.map((item) => `${verdictText[item]}`).join('\n'); + const filesToUploadForSample = Object.keys(sample.files); // Create sample if not exists. try { sample.uuid ??= await dispatch( - createSample(experimentId, name, sampleType, filesToUploadForSample), + createSample( + experimentId, + name, + sampleType, + valid, + validationMessage, + filesToUploadForSample, + ), ); } catch (e) { // If sample creation fails, sample should not be created diff --git a/src/utils/upload/sampleInspector.js b/src/utils/upload/sampleInspector.js new file mode 100644 index 0000000000..aa8d43de17 --- /dev/null +++ b/src/utils/upload/sampleInspector.js @@ -0,0 +1,145 @@ +import { + DecodeUTF8, Decompress, +} from 'fflate'; + +const Verdict = { + INVALID_BARCODES_FILE: 'INVALID_SAMPLE_FILES', + INVALID_FEATURES_FILE: 'INVALID_FEATURES_FILE', + INVALID_SAMPLE_FILE_TRANSPOSED: 'INVALID_SAMPLE_FILE_TRANSPOSED', +}; + +const verdictText = { + [Verdict.INVALID_BARCODES_FILE]: 'Barcodes file is invalid', + [Verdict.INVALID_FEATURES_FILE]: 'Features file is invalid', + [Verdict.INVALID_SAMPLE_FILE_TRANSPOSED]: 'Sample files are transposed', +}; + +const CHUNK_SIZE = 2 ** 18; // 250 kb + +const decodeStream = async (fileSlice) => { + const arrBuffer = await fileSlice.arrayBuffer(); + + let result = ''; + const utfDecode = new DecodeUTF8((data) => { result += data; }); + utfDecode.push(new Uint8Array(arrBuffer)); + + return result; +}; + +const decompressStream = async (fileSlice) => { + const arrBuffer = await fileSlice.arrayBuffer(); + + let result = ''; + const utfDecode = new DecodeUTF8((data) => { result += data; }); + const decompressor = new Decompress((chunk) => { utfDecode.push(chunk); }); + decompressor.push(new Uint8Array(arrBuffer)); + + return result; +}; + +const extractSampleSizes = async (matrix) => { + const { compressed, fileObject } = matrix; + let header = ''; + let matrixHeader = ''; + + const fileSlie = fileObject.slice(0, 500); + + matrixHeader = compressed + ? await decompressStream(fileSlie) + : await decodeStream(fileSlie); + + // The matrix header is the first line in the file that splits into 3 + header = matrixHeader.split('\n').find((line) => line.split(' ').length === 3); + + const [featuresSize, barcodeSize] = header.split(' '); + return { + featuresSize: Number.parseInt(featuresSize, 10), + barcodeSize: Number.parseInt(barcodeSize, 10), + }; +}; + +const countLine = async (fileObject, start, compressed) => { + const end = Math.min(start + CHUNK_SIZE, fileObject.size); + const fileSlice = fileObject.slice(start, end); + + const fileStr = compressed ? await decompressStream(fileSlice) : await decodeStream(fileSlice); + let numLines = (fileStr.match(/\n|\r\n/g) || []).length; + + // Last character might not contain a new line char (\n), so the last line is not counted + // Correct this by adding a line to the count if the last line is not \n + if (fileSlice.size === CHUNK_SIZE) return numLines; + + const lastChar = fileStr[fileStr.length - 1]; + if (lastChar !== '\n') { numLines += 1; } + return numLines; +}; + +const validateFileSize = async (sampleFile, expectedSize) => { + let pointer = 0; + + const counterJobs = []; + + const { compressed, fileObject } = sampleFile; + + while (pointer < fileObject.size) { + counterJobs.push(countLine(fileObject, pointer, compressed)); + pointer += CHUNK_SIZE; + } + + const resultingCounts = await Promise.all(counterJobs); + const numLines = resultingCounts.reduce((count, numLine) => count + numLine, 0); + + return numLines === expectedSize; +}; + +const validateFileSizes = async (sample) => { + const barcodes = sample.files['barcodes.tsv.gz'] || sample.files['barcodes.tsv']; + const features = sample.files['features.tsv.gz'] || sample.files['features.tsv']; + const matrix = sample.files['matrix.mtx.gz'] || sample.files['matrix.mtx']; + + const { barcodeSize, featuresSize } = await extractSampleSizes(matrix); + + const isBarcodeValid = await validateFileSize(barcodes, barcodeSize); + const isFeaturesValid = await validateFileSize(features, featuresSize); + + const valid = isBarcodeValid && isFeaturesValid; + + const verdict = []; + if (!isBarcodeValid) verdict.push(Verdict.INVALID_BARCODES_FILE); + if (!isFeaturesValid) verdict.push(Verdict.INVALID_FEATURES_FILE); + + return { valid, verdict }; +}; + +const validationTests = [ + validateFileSizes, +]; + +const inspectSample = async (sample) => { + console.log('*** sample', sample); + console.log('*** inspecting sample', sample.name); + + // The promises return [{ valid: ..., verdict: ... }, ... ] + const validationPromises = validationTests + .map(async (validationFn) => await validationFn(sample)); + + console.log(validationPromises); + + const result = await Promise.all(validationPromises); + + // This transforms it into { valid: ..., verdict: [...] }, + const { valid, verdict } = result.reduce((acc, curr) => ({ + valid: acc.valid && curr.valid, + verdict: [ + ...acc.verdict, + ...curr.verdict, + ], + }), { valid: true, verdict: [] }); + + return { valid, verdict }; +}; + +export { + inspectSample, + verdictText, +}; From a415931ee662866983eff6f09866d88efdc63364 Mon Sep 17 00:00:00 2001 From: Anugerah Erlaut Date: Fri, 15 Jul 2022 08:02:09 +0100 Subject: [PATCH 02/15] persist sample validit --- src/redux/actions/samples/createSample.js | 4 +++- src/redux/actions/samples/loadSamples.js | 4 ++-- src/utils/upload/processUpload.js | 2 -- src/utils/upload/sampleInspector.js | 5 ----- 4 files changed, 5 insertions(+), 10 deletions(-) diff --git a/src/redux/actions/samples/createSample.js b/src/redux/actions/samples/createSample.js index c8b3beef2f..5988b607ce 100644 --- a/src/redux/actions/samples/createSample.js +++ b/src/redux/actions/samples/createSample.js @@ -69,7 +69,9 @@ const createSample = ( headers: { 'Content-Type': 'application/json', }, - body: JSON.stringify({ name, sampleTechnology }), + body: JSON.stringify({ + name, sampleTechnology, valid, validationMessage, + }), }, ); diff --git a/src/redux/actions/samples/loadSamples.js b/src/redux/actions/samples/loadSamples.js index aecb106334..87b243d0a9 100644 --- a/src/redux/actions/samples/loadSamples.js +++ b/src/redux/actions/samples/loadSamples.js @@ -53,6 +53,8 @@ const toApiV1 = (samples, experimentId) => { metadata: sample.metadata, createdDate: sample.createdAt, name: sample.name, + valid: sample.valid, + validationMessage: sample.validationMessage, lastModified: sample.updatedAt, files: apiV1Files, type: sampleTechnologyConvert(sample.sampleTechnology), @@ -75,8 +77,6 @@ const loadSamples = (experimentId) => async (dispatch) => { const samples = toApiV1(data, experimentId); - // throwIfRequestFailed(response, data, endUserMessages.ERROR_FETCHING_SAMPLES); - dispatch({ type: SAMPLES_LOADED, payload: { diff --git a/src/utils/upload/processUpload.js b/src/utils/upload/processUpload.js index 9ba4b276f4..66689bfb2b 100644 --- a/src/utils/upload/processUpload.js +++ b/src/utils/upload/processUpload.js @@ -130,8 +130,6 @@ const processUpload = async (filesList, sampleType, samples, experimentId, dispa }, {}); Object.entries(samplesMap).forEach(async ([name, sample]) => { - console.log('*** inspecting sample', name); - // Validate sample const { valid, verdict } = await inspectSample(sample); const validationMessage = verdict.map((item) => `${verdictText[item]}`).join('\n'); diff --git a/src/utils/upload/sampleInspector.js b/src/utils/upload/sampleInspector.js index aa8d43de17..624b21ab36 100644 --- a/src/utils/upload/sampleInspector.js +++ b/src/utils/upload/sampleInspector.js @@ -116,15 +116,10 @@ const validationTests = [ ]; const inspectSample = async (sample) => { - console.log('*** sample', sample); - console.log('*** inspecting sample', sample.name); - // The promises return [{ valid: ..., verdict: ... }, ... ] const validationPromises = validationTests .map(async (validationFn) => await validationFn(sample)); - console.log(validationPromises); - const result = await Promise.all(validationPromises); // This transforms it into { valid: ..., verdict: [...] }, From 1d55f82470e4e876c7bfbf720793f5d7fff5fdcb Mon Sep 17 00:00:00 2001 From: Anugerah Erlaut Date: Fri, 15 Jul 2022 08:14:49 +0100 Subject: [PATCH 03/15] disable launching analysis if invalid sample --- src/components/data-management/LaunchAnalysisButton.jsx | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/components/data-management/LaunchAnalysisButton.jsx b/src/components/data-management/LaunchAnalysisButton.jsx index 06f5291caf..b8a6433263 100644 --- a/src/components/data-management/LaunchAnalysisButton.jsx +++ b/src/components/data-management/LaunchAnalysisButton.jsx @@ -113,7 +113,8 @@ const LaunchAnalysisButton = () => { if (!samples[sampleUuid]) return false; const checkedSample = samples[sampleUuid]; - return allSampleFilesUploaded(checkedSample) + return checkedSample.valid + && allSampleFilesUploaded(checkedSample) && allSampleMetadataInserted(checkedSample); }); return canLaunch; @@ -129,7 +130,7 @@ const LaunchAnalysisButton = () => { if (!canLaunchAnalysis()) { return ( {/* disabled button inside tooltip causes tooltip to not function */} {/* https://github.com/react-component/tooltip/issues/18#issuecomment-140078802 */} From df5906e4eade8d79ce8d3df08ddbbe3297572846 Mon Sep 17 00:00:00 2001 From: Anugerah Erlaut Date: Fri, 15 Jul 2022 08:39:12 +0100 Subject: [PATCH 04/15] refactor decode decompress --- src/utils/upload/sampleInspector.js | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/src/utils/upload/sampleInspector.js b/src/utils/upload/sampleInspector.js index 624b21ab36..c24245097d 100644 --- a/src/utils/upload/sampleInspector.js +++ b/src/utils/upload/sampleInspector.js @@ -16,9 +16,7 @@ const verdictText = { const CHUNK_SIZE = 2 ** 18; // 250 kb -const decodeStream = async (fileSlice) => { - const arrBuffer = await fileSlice.arrayBuffer(); - +const decode = async (arrBuffer) => { let result = ''; const utfDecode = new DecodeUTF8((data) => { result += data; }); utfDecode.push(new Uint8Array(arrBuffer)); @@ -26,12 +24,9 @@ const decodeStream = async (fileSlice) => { return result; }; -const decompressStream = async (fileSlice) => { - const arrBuffer = await fileSlice.arrayBuffer(); - +const decompress = async (arrBuffer) => { let result = ''; - const utfDecode = new DecodeUTF8((data) => { result += data; }); - const decompressor = new Decompress((chunk) => { utfDecode.push(chunk); }); + const decompressor = new Decompress((chunk) => { result = chunk; }); decompressor.push(new Uint8Array(arrBuffer)); return result; @@ -42,11 +37,11 @@ const extractSampleSizes = async (matrix) => { let header = ''; let matrixHeader = ''; - const fileSlie = fileObject.slice(0, 500); + const fileArrBuffer = await fileObject.slice(0, 500).arrayBuffer(); matrixHeader = compressed - ? await decompressStream(fileSlie) - : await decodeStream(fileSlie); + ? await decode(await decompress(fileArrBuffer)) + : await decode(fileArrBuffer); // The matrix header is the first line in the file that splits into 3 header = matrixHeader.split('\n').find((line) => line.split(' ').length === 3); @@ -60,14 +55,16 @@ const extractSampleSizes = async (matrix) => { const countLine = async (fileObject, start, compressed) => { const end = Math.min(start + CHUNK_SIZE, fileObject.size); - const fileSlice = fileObject.slice(start, end); + const arrBuffer = await fileObject.slice(start, end).arrayBuffer(); + const fileStr = compressed + ? await decode(await decompress(arrBuffer)) + : await decode(arrBuffer); - const fileStr = compressed ? await decompressStream(fileSlice) : await decodeStream(fileSlice); let numLines = (fileStr.match(/\n|\r\n/g) || []).length; // Last character might not contain a new line char (\n), so the last line is not counted // Correct this by adding a line to the count if the last line is not \n - if (fileSlice.size === CHUNK_SIZE) return numLines; + if (arrBuffer.byteSize === CHUNK_SIZE) return numLines; const lastChar = fileStr[fileStr.length - 1]; if (lastChar !== '\n') { numLines += 1; } From b2347b0852f2bfb692e8b393186fb26552008ed2 Mon Sep 17 00:00:00 2001 From: Anugerah Erlaut Date: Fri, 15 Jul 2022 16:21:31 +0100 Subject: [PATCH 05/15] add transposed error --- .../data-management/SamplesTableCells.jsx | 2 +- src/utils/upload/sampleInspector.js | 29 ++++++++----------- 2 files changed, 13 insertions(+), 18 deletions(-) diff --git a/src/components/data-management/SamplesTableCells.jsx b/src/components/data-management/SamplesTableCells.jsx index 513a0027ef..669155e414 100644 --- a/src/components/data-management/SamplesTableCells.jsx +++ b/src/components/data-management/SamplesTableCells.jsx @@ -178,7 +178,7 @@ const SampleNameCell = (props) => { { !record.valid ? ( - + ) : '' diff --git a/src/utils/upload/sampleInspector.js b/src/utils/upload/sampleInspector.js index c24245097d..80c71bb3fa 100644 --- a/src/utils/upload/sampleInspector.js +++ b/src/utils/upload/sampleInspector.js @@ -60,20 +60,13 @@ const countLine = async (fileObject, start, compressed) => { ? await decode(await decompress(arrBuffer)) : await decode(arrBuffer); - let numLines = (fileStr.match(/\n|\r\n/g) || []).length; + const numLines = (fileStr.match(/\n|\r\n/g) || []).length; - // Last character might not contain a new line char (\n), so the last line is not counted - // Correct this by adding a line to the count if the last line is not \n - if (arrBuffer.byteSize === CHUNK_SIZE) return numLines; - - const lastChar = fileStr[fileStr.length - 1]; - if (lastChar !== '\n') { numLines += 1; } return numLines; }; -const validateFileSize = async (sampleFile, expectedSize) => { +const getNumLines = async (sampleFile) => { let pointer = 0; - const counterJobs = []; const { compressed, fileObject } = sampleFile; @@ -85,8 +78,7 @@ const validateFileSize = async (sampleFile, expectedSize) => { const resultingCounts = await Promise.all(counterJobs); const numLines = resultingCounts.reduce((count, numLine) => count + numLine, 0); - - return numLines === expectedSize; + return numLines; }; const validateFileSizes = async (sample) => { @@ -96,14 +88,17 @@ const validateFileSizes = async (sample) => { const { barcodeSize, featuresSize } = await extractSampleSizes(matrix); - const isBarcodeValid = await validateFileSize(barcodes, barcodeSize); - const isFeaturesValid = await validateFileSize(features, featuresSize); - - const valid = isBarcodeValid && isFeaturesValid; + const numBarcodeLines = await getNumLines(barcodes); + const numFeaturesLines = await getNumLines(features); const verdict = []; - if (!isBarcodeValid) verdict.push(Verdict.INVALID_BARCODES_FILE); - if (!isFeaturesValid) verdict.push(Verdict.INVALID_FEATURES_FILE); + if (numBarcodeLines !== barcodeSize) verdict.push(Verdict.INVALID_BARCODES_FILE); + if (numFeaturesLines !== featuresSize) verdict.push(Verdict.INVALID_FEATURES_FILE); + if (numBarcodeLines === featuresSize && numFeaturesLines === barcodeSize) { + verdict.push(Verdict.INVALID_SAMPLE_FILE_TRANSPOSED); + } + + const valid = numBarcodeLines === barcodeSize && numFeaturesLines === featuresSize; return { valid, verdict }; }; From 1b395fd5145c95f2195b9eba365a6562b704cd93 Mon Sep 17 00:00:00 2001 From: Anugerah Erlaut Date: Fri, 15 Jul 2022 17:20:48 +0100 Subject: [PATCH 06/15] fix tests --- .../LaunchAnalysisButton.test.jsx | 34 +++++++++++++++ .../data/__snapshots__/mockData.test.js.snap | 12 ++++++ .../switchExperiment.test.js.snap | 12 ++++++ .../__snapshots__/createSample.test.js.snap | 12 ++++++ .../__snapshots__/loadSamples.test.js.snap | 2 + .../actions/samples/createSample.test.js | 43 +++++++++++++++++-- .../__snapshots__/samplesReducer.test.js.snap | 28 ++++++++++++ .../__snapshots__/samplesCreate.test.js.snap | 6 +++ .../mockData/generateMockSamples.js | 2 + .../utils/upload/processUpload.test.js | 9 ++++ src/utils/upload/processUpload.js | 1 - 11 files changed, 156 insertions(+), 5 deletions(-) diff --git a/src/__test__/components/data-management/LaunchAnalysisButton.test.jsx b/src/__test__/components/data-management/LaunchAnalysisButton.test.jsx index b62f0bbb96..7be9fc7878 100644 --- a/src/__test__/components/data-management/LaunchAnalysisButton.test.jsx +++ b/src/__test__/components/data-management/LaunchAnalysisButton.test.jsx @@ -96,6 +96,8 @@ const withDataState = { name: sample1Name, experimentId: experiment1id, uuid: sample1Uuid, + valid: true, + validationMessage: '', type: '10X Chromium', metadata: ['value-1'], fileNames: ['features.tsv.gz', 'barcodes.tsv.gz', 'matrix.mtx.gz'], @@ -110,6 +112,8 @@ const withDataState = { name: sample2Name, experimentId: experiment1id, uuid: sample2Uuid, + valid: true, + validationMessage: '', type: '10X Chromium', metadata: ['value-2'], fileNames: ['features.tsv.gz', 'barcodes.tsv.gz', 'matrix.mtx.gz'], @@ -208,6 +212,36 @@ describe('LaunchAnalysisButton', () => { expect(button).toBeDisabled(); }); + it('Process project button is disabled if there is an invalid sample', async () => { + const notAllDataUploaded = { + ...withDataState, + samples: { + ...withDataState.samples, + [sample1Uuid]: { + ...withDataState.samples[sample1Uuid], + valid: false, + validationMessage: 'Invalid file uploaded', + files: { + ...withDataState.samples[sample1Uuid].files, + 'features.tsv.gz': { valid: true, upload: { status: UploadStatus.UPLOADING } }, + }, + }, + }, + }; + + await act(async () => { + render( + + + , + ); + }); + + const button = screen.getByText('Process project').closest('button'); + + expect(button).toBeDisabled(); + }); + it('Process project button is enabled if there is data and all metadata for all samples are uplaoded', async () => { await act(async () => { render( diff --git a/src/__test__/data/__snapshots__/mockData.test.js.snap b/src/__test__/data/__snapshots__/mockData.test.js.snap index 0e0c153af9..fb5b242d0a 100644 --- a/src/__test__/data/__snapshots__/mockData.test.js.snap +++ b/src/__test__/data/__snapshots__/mockData.test.js.snap @@ -57,6 +57,8 @@ Array [ "name": "Mock sample 0", "sampleTechnology": "10x", "updatedAt": "2021-12-07 17:38:42.036+00", + "valid": true, + "validationMessage": "", }, Object { "createdAt": "2021-12-07 17:36:27.773+00", @@ -113,6 +115,8 @@ Array [ "name": "Mock sample 1", "sampleTechnology": "10x", "updatedAt": "2021-12-07 17:38:42.036+00", + "valid": true, + "validationMessage": "", }, Object { "createdAt": "2021-12-07 17:36:27.773+00", @@ -169,6 +173,8 @@ Array [ "name": "Mock sample 2", "sampleTechnology": "10x", "updatedAt": "2021-12-07 17:38:42.036+00", + "valid": true, + "validationMessage": "", }, ] `; @@ -282,6 +288,8 @@ Array [ "name": "Mock sample 0", "sampleTechnology": "10x", "updatedAt": "2021-12-07 17:38:42.036+00", + "valid": true, + "validationMessage": "", }, Object { "createdAt": "2021-12-07 17:36:27.773+00", @@ -338,6 +346,8 @@ Array [ "name": "Mock sample 1", "sampleTechnology": "10x", "updatedAt": "2021-12-07 17:38:42.036+00", + "valid": true, + "validationMessage": "", }, Object { "createdAt": "2021-12-07 17:36:27.773+00", @@ -394,6 +404,8 @@ Array [ "name": "Mock sample 2", "sampleTechnology": "10x", "updatedAt": "2021-12-07 17:38:42.036+00", + "valid": true, + "validationMessage": "", }, ], ] diff --git a/src/__test__/redux/actions/experiments/__snapshots__/switchExperiment.test.js.snap b/src/__test__/redux/actions/experiments/__snapshots__/switchExperiment.test.js.snap index 02b43dc3d0..77020858f2 100644 --- a/src/__test__/redux/actions/experiments/__snapshots__/switchExperiment.test.js.snap +++ b/src/__test__/redux/actions/experiments/__snapshots__/switchExperiment.test.js.snap @@ -233,6 +233,8 @@ Object { "name": "Mock sample 0", "type": "10X Chromium", "uuid": "test9188-d682-test-mock-cb6d644cmock-0", + "valid": true, + "validationMessage": "", }, "test9188-d682-test-mock-cb6d644cmock-1": Object { "createdDate": "2021-12-07 17:36:27.773+00", @@ -276,6 +278,8 @@ Object { "name": "Mock sample 1", "type": "10X Chromium", "uuid": "test9188-d682-test-mock-cb6d644cmock-1", + "valid": true, + "validationMessage": "", }, "test9188-d682-test-mock-cb6d644cmock-2": Object { "createdDate": "2021-12-07 17:36:27.773+00", @@ -319,6 +323,8 @@ Object { "name": "Mock sample 2", "type": "10X Chromium", "uuid": "test9188-d682-test-mock-cb6d644cmock-2", + "valid": true, + "validationMessage": "", }, }, } @@ -557,6 +563,8 @@ Object { "name": "Mock sample 0", "type": "10X Chromium", "uuid": "test9188-d682-test-mock-cb6d644cmock-0", + "valid": true, + "validationMessage": "", }, "test9188-d682-test-mock-cb6d644cmock-1": Object { "createdDate": "2021-12-07 17:36:27.773+00", @@ -600,6 +608,8 @@ Object { "name": "Mock sample 1", "type": "10X Chromium", "uuid": "test9188-d682-test-mock-cb6d644cmock-1", + "valid": true, + "validationMessage": "", }, "test9188-d682-test-mock-cb6d644cmock-2": Object { "createdDate": "2021-12-07 17:36:27.773+00", @@ -643,6 +653,8 @@ Object { "name": "Mock sample 2", "type": "10X Chromium", "uuid": "test9188-d682-test-mock-cb6d644cmock-2", + "valid": true, + "validationMessage": "", }, }, } diff --git a/src/__test__/redux/actions/samples/__snapshots__/createSample.test.js.snap b/src/__test__/redux/actions/samples/__snapshots__/createSample.test.js.snap index 64e7b31c7a..4edfaf27fb 100644 --- a/src/__test__/redux/actions/samples/__snapshots__/createSample.test.js.snap +++ b/src/__test__/redux/actions/samples/__snapshots__/createSample.test.js.snap @@ -15,6 +15,8 @@ exports[`createSample action Works correctly with many files being uploaded 1`] Object { "name": "test sample", "sampleTechnology": "10x", + "valid": true, + "validationMessage": "", } `; @@ -53,6 +55,8 @@ Array [ "name": "test sample", "type": "10X Chromium", "uuid": "abc123", + "valid": true, + "validationMessage": "", }, }, Object { @@ -85,6 +89,8 @@ Array [ "name": "test sample", "type": "10X Chromium", "uuid": "abc123", + "valid": true, + "validationMessage": "", }, }, ] @@ -94,6 +100,8 @@ exports[`createSample action Works correctly with one file being uploaded 1`] = Object { "name": "test sample", "sampleTechnology": "10x", + "valid": true, + "validationMessage": "", } `; @@ -122,6 +130,8 @@ Array [ "name": "test sample", "type": "10X Chromium", "uuid": "abc123", + "valid": true, + "validationMessage": "", }, }, Object { @@ -144,6 +154,8 @@ Array [ "name": "test sample", "type": "10X Chromium", "uuid": "abc123", + "valid": true, + "validationMessage": "", }, }, ] diff --git a/src/__test__/redux/actions/samples/__snapshots__/loadSamples.test.js.snap b/src/__test__/redux/actions/samples/__snapshots__/loadSamples.test.js.snap index ecc9a3fe98..1f9436101c 100644 --- a/src/__test__/redux/actions/samples/__snapshots__/loadSamples.test.js.snap +++ b/src/__test__/redux/actions/samples/__snapshots__/loadSamples.test.js.snap @@ -54,6 +54,8 @@ Object { "name": "BLp7", "type": "10X Chromium", "uuid": "e03ef6ea-5014-4e57-aecd-59964ac9172c", + "valid": undefined, + "validationMessage": undefined, }, }, } diff --git a/src/__test__/redux/actions/samples/createSample.test.js b/src/__test__/redux/actions/samples/createSample.test.js index db00081996..149bbd01af 100644 --- a/src/__test__/redux/actions/samples/createSample.test.js +++ b/src/__test__/redux/actions/samples/createSample.test.js @@ -47,6 +47,9 @@ describe('createSample action', () => { }, }; + const validation = true; + const validationMessage = ''; + let store; beforeEach(() => { @@ -62,7 +65,16 @@ describe('createSample action', () => { it('Works correctly with one file being uploaded', async () => { fetchMock.mockResponse(JSON.stringify({}), { url: 'mockedUrl', status: 200 }); - const newUuid = await store.dispatch(createSample(experimentId, sampleName, mockType, ['matrix.tsv.gz'])); + const newUuid = await store.dispatch( + createSample( + experimentId, + sampleName, + mockType, + validation, + validationMessage, + ['matrix.tsv.gz'], + ), + ); // Returns a new sampleUuid expect(newUuid).toEqual(sampleUuid); @@ -85,7 +97,16 @@ describe('createSample action', () => { it('Works correctly with many files being uploaded', async () => { fetchMock.mockResponse(JSON.stringify({}), { url: 'mockedUrl', status: 200 }); - const newUuid = await store.dispatch(createSample(experimentId, sampleName, mockType, ['matrix.tsv.gz', 'features.tsv.gz', 'barcodes.tsv.gz'])); + const newUuid = await store.dispatch( + createSample( + experimentId, + sampleName, + mockType, + validation, + validationMessage, + ['matrix.tsv.gz', 'features.tsv.gz', 'barcodes.tsv.gz'], + ), + ); // Returns a new sampleUuid expect(newUuid).toEqual(sampleUuid); @@ -110,7 +131,14 @@ describe('createSample action', () => { await expect( store.dispatch( - createSample(experimentId, sampleName, mockType, ['matrix.tsv.gz']), + createSample( + experimentId, + sampleName, + mockType, + validation, + validationMessage, + ['matrix.tsv.gz'], + ), ), ).rejects.toThrow(endUserMessages.ERROR_CREATING_SAMPLE); @@ -125,7 +153,14 @@ describe('createSample action', () => { await expect( store.dispatch( - createSample(experimentId, sampleName, 'unrecognizable type', ['matrix.tsv.gz', 'features.tsv.gz', 'barcodes.tsv.gz']), + createSample( + experimentId, + sampleName, + 'unrecognizable type', + validation, + validationMessage, + ['matrix.tsv.gz', 'features.tsv.gz', 'barcodes.tsv.gz'], + ), ), ).rejects.toThrow('Sample technology unrecognizable type is not recognized'); }); diff --git a/src/__test__/redux/reducers/__snapshots__/samplesReducer.test.js.snap b/src/__test__/redux/reducers/__snapshots__/samplesReducer.test.js.snap index 7fe9d2725b..26e4fb31ed 100644 --- a/src/__test__/redux/reducers/__snapshots__/samplesReducer.test.js.snap +++ b/src/__test__/redux/reducers/__snapshots__/samplesReducer.test.js.snap @@ -14,6 +14,8 @@ Object { "name": "test sample", "type": null, "uuid": "asd123", + "valid": null, + "validationMessage": null, }, "meta": Object { "error": false, @@ -32,6 +34,8 @@ Object { "name": "test sample 2", "type": null, "uuid": "qwe234", + "valid": null, + "validationMessage": null, }, } `; @@ -50,6 +54,8 @@ Object { "name": "test sample", "type": null, "uuid": "asd123", + "valid": null, + "validationMessage": null, }, "meta": Object { "error": false, @@ -78,6 +84,8 @@ Object { "name": "test sample", "type": null, "uuid": "asd123", + "valid": null, + "validationMessage": null, }, "meta": Object { "error": false, @@ -116,6 +124,8 @@ Object { "name": "test sample", "type": null, "uuid": "asd123", + "valid": null, + "validationMessage": null, }, "meta": Object { "error": false, @@ -144,6 +154,8 @@ Object { "name": "test sample", "type": null, "uuid": "asd123", + "valid": null, + "validationMessage": null, }, "meta": Object { "error": false, @@ -167,6 +179,8 @@ Object { "name": "test sample", "type": null, "uuid": "asd123", + "valid": null, + "validationMessage": null, }, "ids": Array [ "asd123", @@ -189,6 +203,8 @@ Object { "name": "test sample 2", "type": null, "uuid": "qwe234", + "valid": null, + "validationMessage": null, }, } `; @@ -207,6 +223,8 @@ Object { "name": "test sample", "type": null, "uuid": "asd123", + "valid": null, + "validationMessage": null, }, "meta": Object { "error": false, @@ -230,6 +248,8 @@ Object { "name": "test sample", "type": null, "uuid": "asd123", + "valid": null, + "validationMessage": null, }, "meta": Object { "error": false, @@ -253,6 +273,8 @@ Object { "name": "test sample", "type": null, "uuid": "asd123", + "valid": null, + "validationMessage": null, }, "meta": Object { "error": "Error message", @@ -276,6 +298,8 @@ Object { "name": "updated name", "type": null, "uuid": "asd123", + "valid": null, + "validationMessage": null, }, "meta": Object { "error": false, @@ -316,6 +340,8 @@ Object { "name": "test sample", "type": null, "uuid": "asd123", + "valid": null, + "validationMessage": null, }, "meta": Object { "error": false, @@ -346,6 +372,8 @@ Object { "name": "test sample", "type": null, "uuid": "asd123", + "valid": null, + "validationMessage": null, }, "meta": Object { "error": false, diff --git a/src/__test__/redux/reducers/samples/__snapshots__/samplesCreate.test.js.snap b/src/__test__/redux/reducers/samples/__snapshots__/samplesCreate.test.js.snap index a6f94f76cc..c6920c8de3 100644 --- a/src/__test__/redux/reducers/samples/__snapshots__/samplesCreate.test.js.snap +++ b/src/__test__/redux/reducers/samples/__snapshots__/samplesCreate.test.js.snap @@ -19,6 +19,8 @@ Object { "name": "oldSampleName", "type": "10x", "uuid": "oldUuid", + "valid": null, + "validationMessage": null, }, "uuid": Object { "complete": false, @@ -32,6 +34,8 @@ Object { "name": "sampleName", "type": "10x", "uuid": "uuid", + "valid": null, + "validationMessage": null, }, } `; @@ -55,6 +59,8 @@ Object { "name": "sampleName", "type": "10x", "uuid": "uuid", + "valid": null, + "validationMessage": null, }, } `; diff --git a/src/__test__/test-utils/mockData/generateMockSamples.js b/src/__test__/test-utils/mockData/generateMockSamples.js index 7352fd7b8d..09047a31af 100644 --- a/src/__test__/test-utils/mockData/generateMockSamples.js +++ b/src/__test__/test-utils/mockData/generateMockSamples.js @@ -7,6 +7,8 @@ const mockSampleTemplate = (experimentId, sampleId, idx) => ({ experimentId, name: `Mock sample ${idx}`, sampleTechnology: '10x', + valid: true, + validationMessage: '', createdAt: '2021-12-07 17:36:27.773+00', updatedAt: '2021-12-07 17:38:42.036+00', metadata: { age: 'BL', timePoint: 'BL' }, diff --git a/src/__test__/utils/upload/processUpload.test.js b/src/__test__/utils/upload/processUpload.test.js index e58bbbf324..45620dba45 100644 --- a/src/__test__/utils/upload/processUpload.test.js +++ b/src/__test__/utils/upload/processUpload.test.js @@ -126,6 +126,15 @@ jest.mock('redux/actions/samples/deleteSamples', () => ({ sendDeleteSamplesRequest: jest.fn(), })); +jest.mock('utils/upload/sampleInspector', () => { + const actual = jest.requireActual('utils/upload/sampleInspector'); + + return { + ...actual, + inspectSample: jest.fn(() => ({ valid: true, verdict: [] })), + }; +}); + let store = null; describe('processUpload', () => { diff --git a/src/utils/upload/processUpload.js b/src/utils/upload/processUpload.js index 66689bfb2b..91cd4f6b59 100644 --- a/src/utils/upload/processUpload.js +++ b/src/utils/upload/processUpload.js @@ -130,7 +130,6 @@ const processUpload = async (filesList, sampleType, samples, experimentId, dispa }, {}); Object.entries(samplesMap).forEach(async ([name, sample]) => { - // Validate sample const { valid, verdict } = await inspectSample(sample); const validationMessage = verdict.map((item) => `${verdictText[item]}`).join('\n'); From ba41a61f091ca8c6f6e728517694fb73780b537c Mon Sep 17 00:00:00 2001 From: Anugerah Erlaut Date: Sat, 16 Jul 2022 11:17:54 +0100 Subject: [PATCH 07/15] add tests for sample checking --- .../SamplesTableCells.test.jsx | 34 ++- src/__test__/data/mock_files/barcodes.tsv | 5 + src/__test__/data/mock_files/barcodes.tsv.gz | Bin 0 -> 84 bytes src/__test__/data/mock_files/features.tsv | 10 + src/__test__/data/mock_files/features.tsv.gz | Bin 0 -> 170 bytes .../data/mock_files/invalid_barcodes.tsv | 3 + .../data/mock_files/invalid_barcodes.tsv.gz | Bin 0 -> 78 bytes .../data/mock_files/invalid_features.tsv | 7 + .../data/mock_files/invalid_features.tsv.gz | Bin 0 -> 154 bytes src/__test__/data/mock_files/matrix.mtx | 3 + src/__test__/data/mock_files/matrix.mtx.gz | Bin 0 -> 136 bytes .../data/mock_files/transposed_matrix.mtx | 3 + .../data/mock_files/transposed_matrix.mtx.gz | Bin 0 -> 153 bytes .../utils/upload/sampleInspector.test.js | 201 ++++++++++++++++++ .../data-management/SamplesTableCells.jsx | 4 +- src/utils/upload/sampleInspector.js | 20 +- 16 files changed, 277 insertions(+), 13 deletions(-) create mode 100644 src/__test__/data/mock_files/barcodes.tsv create mode 100644 src/__test__/data/mock_files/barcodes.tsv.gz create mode 100644 src/__test__/data/mock_files/features.tsv create mode 100644 src/__test__/data/mock_files/features.tsv.gz create mode 100644 src/__test__/data/mock_files/invalid_barcodes.tsv create mode 100644 src/__test__/data/mock_files/invalid_barcodes.tsv.gz create mode 100644 src/__test__/data/mock_files/invalid_features.tsv create mode 100644 src/__test__/data/mock_files/invalid_features.tsv.gz create mode 100644 src/__test__/data/mock_files/matrix.mtx create mode 100644 src/__test__/data/mock_files/matrix.mtx.gz create mode 100644 src/__test__/data/mock_files/transposed_matrix.mtx create mode 100644 src/__test__/data/mock_files/transposed_matrix.mtx.gz create mode 100644 src/__test__/utils/upload/sampleInspector.test.js diff --git a/src/__test__/components/data-management/SamplesTableCells.test.jsx b/src/__test__/components/data-management/SamplesTableCells.test.jsx index 85d2409a70..534f14f055 100644 --- a/src/__test__/components/data-management/SamplesTableCells.test.jsx +++ b/src/__test__/components/data-management/SamplesTableCells.test.jsx @@ -1,5 +1,7 @@ import React from 'react'; -import { screen, render } from '@testing-library/react'; +import { + screen, render, waitFor, +} from '@testing-library/react'; import userEvent from '@testing-library/user-event'; import { act } from 'react-dom/test-utils'; import { Provider } from 'react-redux'; @@ -129,7 +131,7 @@ describe('EditableFieldCell', () => { cellText={mockCellText} dataIndex='mockIndex' rowIdx={1} - onAfterSubmit={() => {}} + onAfterSubmit={() => { }} />); expect(screen.getByText(mockCellText)).toBeInTheDocument(); @@ -148,7 +150,27 @@ describe('SampleNameCell', () => { const cellInfo = { text: mockSampleName, - record: { uuid: 'mock-uuid' }, + record: { uuid: 'mock-uuid', valid: true, validationMessage: '' }, + idx: 1, + }; + + render( + + + , + ); + + expect(screen.getByText(mockSampleName)).toBeInTheDocument(); + expect(screen.queryByLabelText('warning')).toBeNull(); + }); + + it('Shows validation status and message', async () => { + const mockSampleName = 'my mocky name'; + const mockValidationMessage = 'some random error'; + + const cellInfo = { + text: mockSampleName, + record: { uuid: 'mock-uuid', valid: false, validationMessage: mockValidationMessage }, idx: 1, }; @@ -159,5 +181,11 @@ describe('SampleNameCell', () => { ); expect(screen.getByText(mockSampleName)).toBeInTheDocument(); + + userEvent.hover(screen.getByRole('img', { name: 'warning' })); + + await waitFor(() => { + expect(screen.getByText(mockValidationMessage)).toBeInTheDocument(); + }); }); }); diff --git a/src/__test__/data/mock_files/barcodes.tsv b/src/__test__/data/mock_files/barcodes.tsv new file mode 100644 index 0000000000..0de05de1e2 --- /dev/null +++ b/src/__test__/data/mock_files/barcodes.tsv @@ -0,0 +1,5 @@ +AAACCCATCAAACCTG-1 +AAACGAAAGTTGCTGT-1 +AAACGAACACACAGCC-1 +AAACGAATCGTGGGAA-1 +AAACGCTGTTCGGTTA-1 diff --git a/src/__test__/data/mock_files/barcodes.tsv.gz b/src/__test__/data/mock_files/barcodes.tsv.gz new file mode 100644 index 0000000000000000000000000000000000000000..b459d76b2c2a183344db586d3fbea6640067c31e GIT binary patch literal 84 zcmV-a0IUBWiwFpYhtgsI17cxvV{c?-b1rmqb^t}o!3h8`3_{U+cj=)chzA#N|5>Vq q5(4sh5C9}$qG9c`TFS^)#6x^1a>$gW#HW4)Go=f^1g^bb00019(jlb) literal 0 HcmV?d00001 diff --git a/src/__test__/data/mock_files/features.tsv b/src/__test__/data/mock_files/features.tsv new file mode 100644 index 0000000000..982f08c8c6 --- /dev/null +++ b/src/__test__/data/mock_files/features.tsv @@ -0,0 +1,10 @@ +ENSMUSG00000051951 Xkr4 Gene Expression +ENSMUSG00000089699 Gm1992 Gene Expression +ENSMUSG00000102343 Gm37381 Gene Expression +ENSMUSG00000025900 Rp1 Gene Expression +ENSMUSG00000025902 Sox17 Gene Expression +ENSMUSG00000104328 Gm37323 Gene Expression +ENSMUSG00000033845 Mrpl15 Gene Expression +ENSMUSG00000025903 Lypla1 Gene Expression +ENSMUSG00000104217 Gm37988 Gene Expression +ENSMUSG00000033813 Tcea1 Gene Expression diff --git a/src/__test__/data/mock_files/features.tsv.gz b/src/__test__/data/mock_files/features.tsv.gz new file mode 100644 index 0000000000000000000000000000000000000000..aca846216edb7bacf0389647b40cfae303b3470e GIT binary patch literal 170 zcmV;b09F4ViwFoUgVACD17>AmbairNb1rmqb^wizF%AMD5Jl_p6dr+@0hc)dp>Zul z6I){_jO#98oAvghHWmi_^~-zT?LvPHUcF~f48rM_HTXEjwOdk-^L(AgmHXzbY(&7{ zQG}`_Dq_urt<8 literal 0 HcmV?d00001 diff --git a/src/__test__/data/mock_files/invalid_barcodes.tsv b/src/__test__/data/mock_files/invalid_barcodes.tsv new file mode 100644 index 0000000000..9e2abddd25 --- /dev/null +++ b/src/__test__/data/mock_files/invalid_barcodes.tsv @@ -0,0 +1,3 @@ +AAACCCATCAAACCTG-1 +AAACGAAAGTTGCTGT-1 +AAACGAACACACAGCC-1 diff --git a/src/__test__/data/mock_files/invalid_barcodes.tsv.gz b/src/__test__/data/mock_files/invalid_barcodes.tsv.gz new file mode 100644 index 0000000000000000000000000000000000000000..d57ae930fa0c9f6fb1d0ed68b9116d78be513a4b GIT binary patch literal 78 zcmb2|=HSr(b}@;8IWw;;F()%6J}I#%IX@+}Sg)kGjG?%sq@=8@jHj@S#n9FliwFpThtgsI18Ht{VQgt+UuI=tbairNb1rmqb^vqr3-%2Sb~gY6Q$tHr zL(YioA`?#c)Vx#$*NTFo)Z*gI{5&pKWOWvnW|o$m?zx7RmPUA08X6cGn-~LC8k-wi z7~(h7$kfunfHSCo2xUf`!TA-2<^-%ZF*dS*SZ!pC-%Mj;3lmdL-=cyXLsJZu08C^l I93lY#0AhkZ!vFvP literal 0 HcmV?d00001 diff --git a/src/__test__/data/mock_files/matrix.mtx b/src/__test__/data/mock_files/matrix.mtx new file mode 100644 index 0000000000..8c9ebbbb36 --- /dev/null +++ b/src/__test__/data/mock_files/matrix.mtx @@ -0,0 +1,3 @@ +%%MatrixMarket matrix coordinate integer general +%metadata_json: {"format_version": 2, "software_version": "3.1.0"} +10 5 0 diff --git a/src/__test__/data/mock_files/matrix.mtx.gz b/src/__test__/data/mock_files/matrix.mtx.gz new file mode 100644 index 0000000000000000000000000000000000000000..4cf1aa5e019fe687ce1b28cebbb8f8cffb1fbcbe GIT binary patch literal 136 zcmV;30C)c%iwFo1_R(Sh18re+a%p%jZFG15P0K+F!axj0;r*QAConrHQ^ig1&@+@^ z(=uuk$OJ2hcUN4wd>_1=M-H#gk^Lrt?H9P1`COGlP&H%~$0{n$_mnpYo;i5BHq!ww qTFk%O^bl{=lm;9Rpk_<#?DEHS>iVwGJM{%;l=K1VlXSd$00022cRo=7 literal 0 HcmV?d00001 diff --git a/src/__test__/data/mock_files/transposed_matrix.mtx b/src/__test__/data/mock_files/transposed_matrix.mtx new file mode 100644 index 0000000000..b5f33f2ae3 --- /dev/null +++ b/src/__test__/data/mock_files/transposed_matrix.mtx @@ -0,0 +1,3 @@ +%%MatrixMarket matrix coordinate integer general +%metadata_json: {"format_version": 2, "software_version": "3.1.0"} +5 10 1357468 diff --git a/src/__test__/data/mock_files/transposed_matrix.mtx.gz b/src/__test__/data/mock_files/transposed_matrix.mtx.gz new file mode 100644 index 0000000000000000000000000000000000000000..de73480c5b3a39a64158d8358c7141a6558a0fa6 GIT binary patch literal 153 zcmV;K0A~LmiwFpbhtgsI19Wm>ZgX&Nb7f>-ZDDkBX?QMeba((w%RvglKny_9{hT5M zW+$a%txE6EGn8P{GHMgZgesKYU2)~|Klm~BJliGJ1R<@E=u!G|Pu;}D3GX7(1s#MjuhZa%|scA2#^EG?`6=1|= HfdBvi!dFK2 literal 0 HcmV?d00001 diff --git a/src/__test__/utils/upload/sampleInspector.test.js b/src/__test__/utils/upload/sampleInspector.test.js new file mode 100644 index 0000000000..3e8b8c1a1c --- /dev/null +++ b/src/__test__/utils/upload/sampleInspector.test.js @@ -0,0 +1,201 @@ +import { inspectSample, Verdict } from 'utils/upload/sampleInspector'; +import initialState, { sampleFileTemplate, sampleTemplate } from 'redux/reducers/samples/initialState'; + +import * as fs from 'fs'; + +// A function which returns an object to emulate the Blob class +// Required because NodeJS's implementation of Blob modifies the +// data, causing it to be irreproducible in the test +const makeBlob = (data) => ({ + data, + size: data.length, + slice(start, end) { return makeBlob(data.slice(start, end)); }, + arrayBuffer() { return new Promise((resolve) => resolve(data.buffer)); }, +}); + +const mockFileLocations = { + unziped: { + 'barcodes.tsv': 'src/__test__/data/mock_files/barcodes.tsv', + 'features.tsv': 'src/__test__/data/mock_files/features.tsv', + 'matrix.mtx': 'src/__test__/data/mock_files/matrix.mtx', + 'invalid_barcodes.tsv': 'src/__test__/data/mock_files/invalid_barcodes.tsv', + 'invalid_features.tsv': 'src/__test__/data/mock_files/invalid_features.tsv', + 'transposed_matrix.mtx': 'src/__test__/data/mock_files/transposed_matrix.mtx', + }, + zipped: { + 'barcodes.tsv.gz': 'src/__test__/data/mock_files/barcodes.tsv.gz', + 'features.tsv.gz': 'src/__test__/data/mock_files/features.tsv.gz', + 'matrix.mtx.gz': 'src/__test__/data/mock_files/matrix.mtx.gz', + 'invalid_barcodes.tsv.gz': 'src/__test__/data/mock_files/invalid_barcodes.tsv.gz', + 'invalid_features.tsv.gz': 'src/__test__/data/mock_files/invalid_features.tsv.gz', + 'transposed_matrix.mtx.gz': 'src/__test__/data/mock_files/transposed_matrix.mtx.gz', + }, +}; + +// Manually resolve fflate import for Jest the module definition for fflate +// because it does not correctly resolve to the intended module +jest.mock('fflate', () => { + const realModule = jest.requireActual('../../../../node_modules/fflate/umd/index.js'); + + return { + _esModule: true, + ...realModule, + }; +}); + +const prepareMockFiles = (fileLocations) => { + const result = {}; + + // Read and prepare each file object + Object.entries(fileLocations).forEach(([filename, location]) => { + const fileUint8Arr = new Uint8Array(fs.readFileSync(location)); + result[filename] = makeBlob(fileUint8Arr); + }); + + return result; +}; + +const mockUnzippedFileObjects = prepareMockFiles(mockFileLocations.unziped); +const mockZippedFileObjects = prepareMockFiles(mockFileLocations.zipped); + +jest.mock('utils/upload/readFileToBuffer'); + +const mockZippedSample = { + ...sampleTemplate, + ...initialState, + name: 'mockZippedSample', + fileNames: [ + 'features.tsv.gz', + 'barcodes.tsv.gz', + 'matrix.mtx.gz', + ], + files: { + 'features.tsv.gz': { + ...sampleFileTemplate, + name: 'features.tsv.gz', + fileObject: mockZippedFileObjects['features.tsv.gz'], + size: mockZippedFileObjects['features.tsv.gz'].size, + path: '/transposed/features.tsv.gz', + compressed: true, + }, + 'barcodes.tsv.gz': { + ...sampleFileTemplate, + name: 'barcodes.tsv.gz', + fileObject: mockZippedFileObjects['barcodes.tsv.gz'], + size: mockZippedFileObjects['barcodes.tsv.gz'].size, + path: '/transposed/barcodes.tsv.gz', + compressed: true, + }, + 'matrix.mtx.gz': { + ...sampleFileTemplate, + name: 'matrix.mtx.gz', + fileObject: mockZippedFileObjects['matrix.mtx.gz'], + size: mockZippedFileObjects['matrix.mtx.gz'].size, + path: '/transposed/matrix.mtx.gz', + compressed: true, + }, + }, +}; + +const mockUnzippedSample = { + ...sampleTemplate, + ...initialState, + name: 'mockUnzippedSample', + fileNames: [ + 'features.tsv', + 'barcodes.tsv', + 'matrix.mtx', + ], + files: { + 'features.tsv': { + ...sampleFileTemplate, + name: 'features.tsv', + fileObject: mockUnzippedFileObjects['features.tsv'], + size: mockUnzippedFileObjects['features.tsv'].size, + path: '/transposed/features.tsv', + compressed: false, + }, + 'barcodes.tsv': { + ...sampleFileTemplate, + name: 'barcodes.tsv', + fileObject: mockUnzippedFileObjects['barcodes.tsv'], + size: mockUnzippedFileObjects['barcodes.tsv'].size, + path: '/transposed/barcodes.tsv', + compressed: false, + }, + 'matrix.mtx': { + ...sampleFileTemplate, + name: 'matrix.mtx', + fileObject: mockUnzippedFileObjects['matrix.mtx'], + size: mockUnzippedFileObjects['matrix.mtx'].size, + path: '/transposed/matrix.mtx', + compressed: false, + }, + }, +}; + +describe('sampleInspector', () => { + it('Correctly pass valid zipped samples', async () => { + const result = await inspectSample(mockZippedSample); + expect(result).toEqual({ valid: true, verdict: [] }); + }); + + it('Correctly pass valid unzipped samples', async () => { + const result = await inspectSample(mockUnzippedSample); + expect(result).toEqual({ valid: true, verdict: [] }); + }); + + it('Correctly identifies invalid barcodes file', async () => { + const mockInvalidBarcodesFile = { + ...mockZippedSample, + files: { + ...mockZippedSample.files, + 'barcodes.tsv.gz': { + ...mockZippedSample.files['barcodes.tsv.gz'], + fileObject: mockZippedFileObjects['invalid_barcodes.tsv.gz'], + size: mockZippedFileObjects['invalid_barcodes.tsv.gz'].size, + }, + }, + }; + + const result = await inspectSample(mockInvalidBarcodesFile); + expect(result).toEqual({ valid: false, verdict: [Verdict.INVALID_BARCODES_FILE] }); + }); + + it('Correctly identifies invalid features file', async () => { + const mockInvalidFeaturesFile = { + ...mockZippedSample, + files: { + ...mockZippedSample.files, + 'features.tsv.gz': { + ...mockZippedSample.files['features.tsv.gz'], + fileObject: mockZippedFileObjects['invalid_features.tsv.gz'], + size: mockZippedFileObjects['invalid_features.tsv.gz'].size, + }, + }, + }; + + const result = await inspectSample(mockInvalidFeaturesFile); + expect(result).toEqual({ valid: false, verdict: [Verdict.INVALID_FEATURES_FILE] }); + }); + + it('Correctly identifies transposed matrix file', async () => { + const mockTransposedFile = { + ...mockZippedSample, + files: { + ...mockZippedSample.files, + 'matrix.mtx.gz': { + ...mockZippedSample.files['matrix.mtx.gz'], + fileObject: mockZippedFileObjects['transposed_matrix.mtx.gz'], + size: mockZippedFileObjects['transposed_matrix.mtx.gz'].size, + }, + }, + }; + + const result = await inspectSample(mockTransposedFile); + expect(result).toEqual({ + valid: false, + verdict: [Verdict.INVALID_TRANSPOSED_MATRIX], + }); + }); +}); diff --git a/src/components/data-management/SamplesTableCells.jsx b/src/components/data-management/SamplesTableCells.jsx index 669155e414..46cb006399 100644 --- a/src/components/data-management/SamplesTableCells.jsx +++ b/src/components/data-management/SamplesTableCells.jsx @@ -177,11 +177,11 @@ const SampleNameCell = (props) => { /> { - !record.valid ? ( + !record.valid && ( - ) : '' + ) } ); diff --git a/src/utils/upload/sampleInspector.js b/src/utils/upload/sampleInspector.js index 80c71bb3fa..a18fcb8227 100644 --- a/src/utils/upload/sampleInspector.js +++ b/src/utils/upload/sampleInspector.js @@ -5,13 +5,13 @@ import { const Verdict = { INVALID_BARCODES_FILE: 'INVALID_SAMPLE_FILES', INVALID_FEATURES_FILE: 'INVALID_FEATURES_FILE', - INVALID_SAMPLE_FILE_TRANSPOSED: 'INVALID_SAMPLE_FILE_TRANSPOSED', + INVALID_TRANSPOSED_MATRIX: 'INVALID_TRANSPOSED_MATRIX', }; const verdictText = { [Verdict.INVALID_BARCODES_FILE]: 'Barcodes file is invalid', [Verdict.INVALID_FEATURES_FILE]: 'Features file is invalid', - [Verdict.INVALID_SAMPLE_FILE_TRANSPOSED]: 'Sample files are transposed', + [Verdict.INVALID_TRANSPOSED_MATRIX]: 'Sample files are transposed', }; const CHUNK_SIZE = 2 ** 18; // 250 kb @@ -92,14 +92,17 @@ const validateFileSizes = async (sample) => { const numFeaturesLines = await getNumLines(features); const verdict = []; - if (numBarcodeLines !== barcodeSize) verdict.push(Verdict.INVALID_BARCODES_FILE); - if (numFeaturesLines !== featuresSize) verdict.push(Verdict.INVALID_FEATURES_FILE); - if (numBarcodeLines === featuresSize && numFeaturesLines === barcodeSize) { - verdict.push(Verdict.INVALID_SAMPLE_FILE_TRANSPOSED); - } - const valid = numBarcodeLines === barcodeSize && numFeaturesLines === featuresSize; + const isSampleTransposed = numBarcodeLines === featuresSize && numFeaturesLines === barcodeSize; + const isBarcodesInvalid = numBarcodeLines !== barcodeSize; + const isFeaturesInvalid = numFeaturesLines !== featuresSize; + + if (isSampleTransposed) { verdict.push(Verdict.INVALID_TRANSPOSED_MATRIX); } else { + if (isBarcodesInvalid) verdict.push(Verdict.INVALID_BARCODES_FILE); + if (isFeaturesInvalid) verdict.push(Verdict.INVALID_FEATURES_FILE); + } + return { valid, verdict }; }; @@ -128,5 +131,6 @@ const inspectSample = async (sample) => { export { inspectSample, + Verdict, verdictText, }; From cf011e4269bd891c37309cbea40320814b9ab1e5 Mon Sep 17 00:00:00 2001 From: Anugerah Erlaut Date: Mon, 18 Jul 2022 14:14:48 +0100 Subject: [PATCH 08/15] revert upload invalid sample --- .../LaunchAnalysisButton.test.jsx | 34 --------------- .../SamplesTableCells.test.jsx | 32 +------------- .../data/__snapshots__/mockData.test.js.snap | 12 ------ .../switchExperiment.test.js.snap | 12 ------ .../__snapshots__/createSample.test.js.snap | 12 ------ .../__snapshots__/loadSamples.test.js.snap | 2 - .../actions/samples/createSample.test.js | 43 ++----------------- .../__snapshots__/samplesReducer.test.js.snap | 28 ------------ .../__snapshots__/samplesCreate.test.js.snap | 6 --- .../mockData/generateMockSamples.js | 2 - .../data-management/LaunchAnalysisButton.jsx | 5 +-- .../data-management/SamplesTable.jsx | 6 +-- .../data-management/SamplesTableCells.jsx | 11 +---- src/redux/actions/samples/createSample.js | 8 +--- src/redux/actions/samples/loadSamples.js | 2 - src/redux/reducers/samples/initialState.js | 2 - src/utils/upload/processUpload.js | 8 +++- src/utils/upload/sampleInspector.js | 2 +- 18 files changed, 18 insertions(+), 209 deletions(-) diff --git a/src/__test__/components/data-management/LaunchAnalysisButton.test.jsx b/src/__test__/components/data-management/LaunchAnalysisButton.test.jsx index 7be9fc7878..b62f0bbb96 100644 --- a/src/__test__/components/data-management/LaunchAnalysisButton.test.jsx +++ b/src/__test__/components/data-management/LaunchAnalysisButton.test.jsx @@ -96,8 +96,6 @@ const withDataState = { name: sample1Name, experimentId: experiment1id, uuid: sample1Uuid, - valid: true, - validationMessage: '', type: '10X Chromium', metadata: ['value-1'], fileNames: ['features.tsv.gz', 'barcodes.tsv.gz', 'matrix.mtx.gz'], @@ -112,8 +110,6 @@ const withDataState = { name: sample2Name, experimentId: experiment1id, uuid: sample2Uuid, - valid: true, - validationMessage: '', type: '10X Chromium', metadata: ['value-2'], fileNames: ['features.tsv.gz', 'barcodes.tsv.gz', 'matrix.mtx.gz'], @@ -212,36 +208,6 @@ describe('LaunchAnalysisButton', () => { expect(button).toBeDisabled(); }); - it('Process project button is disabled if there is an invalid sample', async () => { - const notAllDataUploaded = { - ...withDataState, - samples: { - ...withDataState.samples, - [sample1Uuid]: { - ...withDataState.samples[sample1Uuid], - valid: false, - validationMessage: 'Invalid file uploaded', - files: { - ...withDataState.samples[sample1Uuid].files, - 'features.tsv.gz': { valid: true, upload: { status: UploadStatus.UPLOADING } }, - }, - }, - }, - }; - - await act(async () => { - render( - - - , - ); - }); - - const button = screen.getByText('Process project').closest('button'); - - expect(button).toBeDisabled(); - }); - it('Process project button is enabled if there is data and all metadata for all samples are uplaoded', async () => { await act(async () => { render( diff --git a/src/__test__/components/data-management/SamplesTableCells.test.jsx b/src/__test__/components/data-management/SamplesTableCells.test.jsx index 534f14f055..4b0fc98c19 100644 --- a/src/__test__/components/data-management/SamplesTableCells.test.jsx +++ b/src/__test__/components/data-management/SamplesTableCells.test.jsx @@ -1,7 +1,5 @@ import React from 'react'; -import { - screen, render, waitFor, -} from '@testing-library/react'; +import { screen, render } from '@testing-library/react'; import userEvent from '@testing-library/user-event'; import { act } from 'react-dom/test-utils'; import { Provider } from 'react-redux'; @@ -150,27 +148,7 @@ describe('SampleNameCell', () => { const cellInfo = { text: mockSampleName, - record: { uuid: 'mock-uuid', valid: true, validationMessage: '' }, - idx: 1, - }; - - render( - - - , - ); - - expect(screen.getByText(mockSampleName)).toBeInTheDocument(); - expect(screen.queryByLabelText('warning')).toBeNull(); - }); - - it('Shows validation status and message', async () => { - const mockSampleName = 'my mocky name'; - const mockValidationMessage = 'some random error'; - - const cellInfo = { - text: mockSampleName, - record: { uuid: 'mock-uuid', valid: false, validationMessage: mockValidationMessage }, + record: { uuid: 'mock-uuid' }, idx: 1, }; @@ -181,11 +159,5 @@ describe('SampleNameCell', () => { ); expect(screen.getByText(mockSampleName)).toBeInTheDocument(); - - userEvent.hover(screen.getByRole('img', { name: 'warning' })); - - await waitFor(() => { - expect(screen.getByText(mockValidationMessage)).toBeInTheDocument(); - }); }); }); diff --git a/src/__test__/data/__snapshots__/mockData.test.js.snap b/src/__test__/data/__snapshots__/mockData.test.js.snap index fb5b242d0a..0e0c153af9 100644 --- a/src/__test__/data/__snapshots__/mockData.test.js.snap +++ b/src/__test__/data/__snapshots__/mockData.test.js.snap @@ -57,8 +57,6 @@ Array [ "name": "Mock sample 0", "sampleTechnology": "10x", "updatedAt": "2021-12-07 17:38:42.036+00", - "valid": true, - "validationMessage": "", }, Object { "createdAt": "2021-12-07 17:36:27.773+00", @@ -115,8 +113,6 @@ Array [ "name": "Mock sample 1", "sampleTechnology": "10x", "updatedAt": "2021-12-07 17:38:42.036+00", - "valid": true, - "validationMessage": "", }, Object { "createdAt": "2021-12-07 17:36:27.773+00", @@ -173,8 +169,6 @@ Array [ "name": "Mock sample 2", "sampleTechnology": "10x", "updatedAt": "2021-12-07 17:38:42.036+00", - "valid": true, - "validationMessage": "", }, ] `; @@ -288,8 +282,6 @@ Array [ "name": "Mock sample 0", "sampleTechnology": "10x", "updatedAt": "2021-12-07 17:38:42.036+00", - "valid": true, - "validationMessage": "", }, Object { "createdAt": "2021-12-07 17:36:27.773+00", @@ -346,8 +338,6 @@ Array [ "name": "Mock sample 1", "sampleTechnology": "10x", "updatedAt": "2021-12-07 17:38:42.036+00", - "valid": true, - "validationMessage": "", }, Object { "createdAt": "2021-12-07 17:36:27.773+00", @@ -404,8 +394,6 @@ Array [ "name": "Mock sample 2", "sampleTechnology": "10x", "updatedAt": "2021-12-07 17:38:42.036+00", - "valid": true, - "validationMessage": "", }, ], ] diff --git a/src/__test__/redux/actions/experiments/__snapshots__/switchExperiment.test.js.snap b/src/__test__/redux/actions/experiments/__snapshots__/switchExperiment.test.js.snap index 77020858f2..02b43dc3d0 100644 --- a/src/__test__/redux/actions/experiments/__snapshots__/switchExperiment.test.js.snap +++ b/src/__test__/redux/actions/experiments/__snapshots__/switchExperiment.test.js.snap @@ -233,8 +233,6 @@ Object { "name": "Mock sample 0", "type": "10X Chromium", "uuid": "test9188-d682-test-mock-cb6d644cmock-0", - "valid": true, - "validationMessage": "", }, "test9188-d682-test-mock-cb6d644cmock-1": Object { "createdDate": "2021-12-07 17:36:27.773+00", @@ -278,8 +276,6 @@ Object { "name": "Mock sample 1", "type": "10X Chromium", "uuid": "test9188-d682-test-mock-cb6d644cmock-1", - "valid": true, - "validationMessage": "", }, "test9188-d682-test-mock-cb6d644cmock-2": Object { "createdDate": "2021-12-07 17:36:27.773+00", @@ -323,8 +319,6 @@ Object { "name": "Mock sample 2", "type": "10X Chromium", "uuid": "test9188-d682-test-mock-cb6d644cmock-2", - "valid": true, - "validationMessage": "", }, }, } @@ -563,8 +557,6 @@ Object { "name": "Mock sample 0", "type": "10X Chromium", "uuid": "test9188-d682-test-mock-cb6d644cmock-0", - "valid": true, - "validationMessage": "", }, "test9188-d682-test-mock-cb6d644cmock-1": Object { "createdDate": "2021-12-07 17:36:27.773+00", @@ -608,8 +600,6 @@ Object { "name": "Mock sample 1", "type": "10X Chromium", "uuid": "test9188-d682-test-mock-cb6d644cmock-1", - "valid": true, - "validationMessage": "", }, "test9188-d682-test-mock-cb6d644cmock-2": Object { "createdDate": "2021-12-07 17:36:27.773+00", @@ -653,8 +643,6 @@ Object { "name": "Mock sample 2", "type": "10X Chromium", "uuid": "test9188-d682-test-mock-cb6d644cmock-2", - "valid": true, - "validationMessage": "", }, }, } diff --git a/src/__test__/redux/actions/samples/__snapshots__/createSample.test.js.snap b/src/__test__/redux/actions/samples/__snapshots__/createSample.test.js.snap index 4edfaf27fb..64e7b31c7a 100644 --- a/src/__test__/redux/actions/samples/__snapshots__/createSample.test.js.snap +++ b/src/__test__/redux/actions/samples/__snapshots__/createSample.test.js.snap @@ -15,8 +15,6 @@ exports[`createSample action Works correctly with many files being uploaded 1`] Object { "name": "test sample", "sampleTechnology": "10x", - "valid": true, - "validationMessage": "", } `; @@ -55,8 +53,6 @@ Array [ "name": "test sample", "type": "10X Chromium", "uuid": "abc123", - "valid": true, - "validationMessage": "", }, }, Object { @@ -89,8 +85,6 @@ Array [ "name": "test sample", "type": "10X Chromium", "uuid": "abc123", - "valid": true, - "validationMessage": "", }, }, ] @@ -100,8 +94,6 @@ exports[`createSample action Works correctly with one file being uploaded 1`] = Object { "name": "test sample", "sampleTechnology": "10x", - "valid": true, - "validationMessage": "", } `; @@ -130,8 +122,6 @@ Array [ "name": "test sample", "type": "10X Chromium", "uuid": "abc123", - "valid": true, - "validationMessage": "", }, }, Object { @@ -154,8 +144,6 @@ Array [ "name": "test sample", "type": "10X Chromium", "uuid": "abc123", - "valid": true, - "validationMessage": "", }, }, ] diff --git a/src/__test__/redux/actions/samples/__snapshots__/loadSamples.test.js.snap b/src/__test__/redux/actions/samples/__snapshots__/loadSamples.test.js.snap index 1f9436101c..ecc9a3fe98 100644 --- a/src/__test__/redux/actions/samples/__snapshots__/loadSamples.test.js.snap +++ b/src/__test__/redux/actions/samples/__snapshots__/loadSamples.test.js.snap @@ -54,8 +54,6 @@ Object { "name": "BLp7", "type": "10X Chromium", "uuid": "e03ef6ea-5014-4e57-aecd-59964ac9172c", - "valid": undefined, - "validationMessage": undefined, }, }, } diff --git a/src/__test__/redux/actions/samples/createSample.test.js b/src/__test__/redux/actions/samples/createSample.test.js index 149bbd01af..db00081996 100644 --- a/src/__test__/redux/actions/samples/createSample.test.js +++ b/src/__test__/redux/actions/samples/createSample.test.js @@ -47,9 +47,6 @@ describe('createSample action', () => { }, }; - const validation = true; - const validationMessage = ''; - let store; beforeEach(() => { @@ -65,16 +62,7 @@ describe('createSample action', () => { it('Works correctly with one file being uploaded', async () => { fetchMock.mockResponse(JSON.stringify({}), { url: 'mockedUrl', status: 200 }); - const newUuid = await store.dispatch( - createSample( - experimentId, - sampleName, - mockType, - validation, - validationMessage, - ['matrix.tsv.gz'], - ), - ); + const newUuid = await store.dispatch(createSample(experimentId, sampleName, mockType, ['matrix.tsv.gz'])); // Returns a new sampleUuid expect(newUuid).toEqual(sampleUuid); @@ -97,16 +85,7 @@ describe('createSample action', () => { it('Works correctly with many files being uploaded', async () => { fetchMock.mockResponse(JSON.stringify({}), { url: 'mockedUrl', status: 200 }); - const newUuid = await store.dispatch( - createSample( - experimentId, - sampleName, - mockType, - validation, - validationMessage, - ['matrix.tsv.gz', 'features.tsv.gz', 'barcodes.tsv.gz'], - ), - ); + const newUuid = await store.dispatch(createSample(experimentId, sampleName, mockType, ['matrix.tsv.gz', 'features.tsv.gz', 'barcodes.tsv.gz'])); // Returns a new sampleUuid expect(newUuid).toEqual(sampleUuid); @@ -131,14 +110,7 @@ describe('createSample action', () => { await expect( store.dispatch( - createSample( - experimentId, - sampleName, - mockType, - validation, - validationMessage, - ['matrix.tsv.gz'], - ), + createSample(experimentId, sampleName, mockType, ['matrix.tsv.gz']), ), ).rejects.toThrow(endUserMessages.ERROR_CREATING_SAMPLE); @@ -153,14 +125,7 @@ describe('createSample action', () => { await expect( store.dispatch( - createSample( - experimentId, - sampleName, - 'unrecognizable type', - validation, - validationMessage, - ['matrix.tsv.gz', 'features.tsv.gz', 'barcodes.tsv.gz'], - ), + createSample(experimentId, sampleName, 'unrecognizable type', ['matrix.tsv.gz', 'features.tsv.gz', 'barcodes.tsv.gz']), ), ).rejects.toThrow('Sample technology unrecognizable type is not recognized'); }); diff --git a/src/__test__/redux/reducers/__snapshots__/samplesReducer.test.js.snap b/src/__test__/redux/reducers/__snapshots__/samplesReducer.test.js.snap index 26e4fb31ed..7fe9d2725b 100644 --- a/src/__test__/redux/reducers/__snapshots__/samplesReducer.test.js.snap +++ b/src/__test__/redux/reducers/__snapshots__/samplesReducer.test.js.snap @@ -14,8 +14,6 @@ Object { "name": "test sample", "type": null, "uuid": "asd123", - "valid": null, - "validationMessage": null, }, "meta": Object { "error": false, @@ -34,8 +32,6 @@ Object { "name": "test sample 2", "type": null, "uuid": "qwe234", - "valid": null, - "validationMessage": null, }, } `; @@ -54,8 +50,6 @@ Object { "name": "test sample", "type": null, "uuid": "asd123", - "valid": null, - "validationMessage": null, }, "meta": Object { "error": false, @@ -84,8 +78,6 @@ Object { "name": "test sample", "type": null, "uuid": "asd123", - "valid": null, - "validationMessage": null, }, "meta": Object { "error": false, @@ -124,8 +116,6 @@ Object { "name": "test sample", "type": null, "uuid": "asd123", - "valid": null, - "validationMessage": null, }, "meta": Object { "error": false, @@ -154,8 +144,6 @@ Object { "name": "test sample", "type": null, "uuid": "asd123", - "valid": null, - "validationMessage": null, }, "meta": Object { "error": false, @@ -179,8 +167,6 @@ Object { "name": "test sample", "type": null, "uuid": "asd123", - "valid": null, - "validationMessage": null, }, "ids": Array [ "asd123", @@ -203,8 +189,6 @@ Object { "name": "test sample 2", "type": null, "uuid": "qwe234", - "valid": null, - "validationMessage": null, }, } `; @@ -223,8 +207,6 @@ Object { "name": "test sample", "type": null, "uuid": "asd123", - "valid": null, - "validationMessage": null, }, "meta": Object { "error": false, @@ -248,8 +230,6 @@ Object { "name": "test sample", "type": null, "uuid": "asd123", - "valid": null, - "validationMessage": null, }, "meta": Object { "error": false, @@ -273,8 +253,6 @@ Object { "name": "test sample", "type": null, "uuid": "asd123", - "valid": null, - "validationMessage": null, }, "meta": Object { "error": "Error message", @@ -298,8 +276,6 @@ Object { "name": "updated name", "type": null, "uuid": "asd123", - "valid": null, - "validationMessage": null, }, "meta": Object { "error": false, @@ -340,8 +316,6 @@ Object { "name": "test sample", "type": null, "uuid": "asd123", - "valid": null, - "validationMessage": null, }, "meta": Object { "error": false, @@ -372,8 +346,6 @@ Object { "name": "test sample", "type": null, "uuid": "asd123", - "valid": null, - "validationMessage": null, }, "meta": Object { "error": false, diff --git a/src/__test__/redux/reducers/samples/__snapshots__/samplesCreate.test.js.snap b/src/__test__/redux/reducers/samples/__snapshots__/samplesCreate.test.js.snap index c6920c8de3..a6f94f76cc 100644 --- a/src/__test__/redux/reducers/samples/__snapshots__/samplesCreate.test.js.snap +++ b/src/__test__/redux/reducers/samples/__snapshots__/samplesCreate.test.js.snap @@ -19,8 +19,6 @@ Object { "name": "oldSampleName", "type": "10x", "uuid": "oldUuid", - "valid": null, - "validationMessage": null, }, "uuid": Object { "complete": false, @@ -34,8 +32,6 @@ Object { "name": "sampleName", "type": "10x", "uuid": "uuid", - "valid": null, - "validationMessage": null, }, } `; @@ -59,8 +55,6 @@ Object { "name": "sampleName", "type": "10x", "uuid": "uuid", - "valid": null, - "validationMessage": null, }, } `; diff --git a/src/__test__/test-utils/mockData/generateMockSamples.js b/src/__test__/test-utils/mockData/generateMockSamples.js index 09047a31af..7352fd7b8d 100644 --- a/src/__test__/test-utils/mockData/generateMockSamples.js +++ b/src/__test__/test-utils/mockData/generateMockSamples.js @@ -7,8 +7,6 @@ const mockSampleTemplate = (experimentId, sampleId, idx) => ({ experimentId, name: `Mock sample ${idx}`, sampleTechnology: '10x', - valid: true, - validationMessage: '', createdAt: '2021-12-07 17:36:27.773+00', updatedAt: '2021-12-07 17:38:42.036+00', metadata: { age: 'BL', timePoint: 'BL' }, diff --git a/src/components/data-management/LaunchAnalysisButton.jsx b/src/components/data-management/LaunchAnalysisButton.jsx index b8a6433263..06f5291caf 100644 --- a/src/components/data-management/LaunchAnalysisButton.jsx +++ b/src/components/data-management/LaunchAnalysisButton.jsx @@ -113,8 +113,7 @@ const LaunchAnalysisButton = () => { if (!samples[sampleUuid]) return false; const checkedSample = samples[sampleUuid]; - return checkedSample.valid - && allSampleFilesUploaded(checkedSample) + return allSampleFilesUploaded(checkedSample) && allSampleMetadataInserted(checkedSample); }); return canLaunch; @@ -130,7 +129,7 @@ const LaunchAnalysisButton = () => { if (!canLaunchAnalysis()) { return ( {/* disabled button inside tooltip causes tooltip to not function */} {/* https://github.com/react-component/tooltip/issues/18#issuecomment-140078802 */} diff --git a/src/components/data-management/SamplesTable.jsx b/src/components/data-management/SamplesTable.jsx index 3bcbd7f2d1..2abf47990e 100644 --- a/src/components/data-management/SamplesTable.jsx +++ b/src/components/data-management/SamplesTable.jsx @@ -230,14 +230,10 @@ const SamplesTable = forwardRef((props, ref) => { const genesData = { sampleUuid, file: genesFile }; const matrixData = { sampleUuid, file: matrixFile }; - const { name, valid, validationMessage } = samples[sampleUuid] ?? {}; - return { key: idx, - name: name || 'UPLOAD ERROR: Please reupload sample', + name: samples[sampleUuid]?.name || 'UPLOAD ERROR: Please reupload sample', uuid: sampleUuid, - valid: valid || false, - validationMessage: validationMessage || '', barcodes: barcodesData, genes: genesData, matrix: matrixData, diff --git a/src/components/data-management/SamplesTableCells.jsx b/src/components/data-management/SamplesTableCells.jsx index 46cb006399..17ac9bbfa2 100644 --- a/src/components/data-management/SamplesTableCells.jsx +++ b/src/components/data-management/SamplesTableCells.jsx @@ -2,9 +2,7 @@ import React, { useRef, useState } from 'react'; import { Space, Typography, Progress, Tooltip, Button, } from 'antd'; -import { - UploadOutlined, WarningFilled, -} from '@ant-design/icons'; +import { UploadOutlined } from '@ant-design/icons'; import { useDispatch } from 'react-redux'; import PropTypes from 'prop-types'; import styled from 'styled-components'; @@ -176,13 +174,6 @@ const SampleNameCell = (props) => { onDelete={() => dispatch(deleteSamples([record.uuid]))} /> - { - !record.valid && ( - - - - ) - } ); }; diff --git a/src/redux/actions/samples/createSample.js b/src/redux/actions/samples/createSample.js index 5988b607ce..0b6dbcd548 100644 --- a/src/redux/actions/samples/createSample.js +++ b/src/redux/actions/samples/createSample.js @@ -18,8 +18,6 @@ const createSample = ( experimentId, name, type, - valid, - validationMessage, filesToUpload, ) => async (dispatch, getState) => { const experiment = getState().experiments[experimentId]; @@ -40,8 +38,6 @@ const createSample = ( type, experimentId, uuid: newSampleUuid, - valid, - validationMessage, createdDate, lastModified: createdDate, metadata: experiment?.metadataKeys @@ -69,9 +65,7 @@ const createSample = ( headers: { 'Content-Type': 'application/json', }, - body: JSON.stringify({ - name, sampleTechnology, valid, validationMessage, - }), + body: JSON.stringify({ name, sampleTechnology }), }, ); diff --git a/src/redux/actions/samples/loadSamples.js b/src/redux/actions/samples/loadSamples.js index 87b243d0a9..d8416f10f4 100644 --- a/src/redux/actions/samples/loadSamples.js +++ b/src/redux/actions/samples/loadSamples.js @@ -53,8 +53,6 @@ const toApiV1 = (samples, experimentId) => { metadata: sample.metadata, createdDate: sample.createdAt, name: sample.name, - valid: sample.valid, - validationMessage: sample.validationMessage, lastModified: sample.updatedAt, files: apiV1Files, type: sampleTechnologyConvert(sample.sampleTechnology), diff --git a/src/redux/reducers/samples/initialState.js b/src/redux/reducers/samples/initialState.js index b1c970db0b..2b83ad2c68 100644 --- a/src/redux/reducers/samples/initialState.js +++ b/src/redux/reducers/samples/initialState.js @@ -3,8 +3,6 @@ const sampleTemplate = { experimentId: null, uuid: null, type: null, - valid: null, - validationMessage: null, createdDate: null, lastModified: null, complete: false, diff --git a/src/utils/upload/processUpload.js b/src/utils/upload/processUpload.js index 91cd4f6b59..cf3770bcbc 100644 --- a/src/utils/upload/processUpload.js +++ b/src/utils/upload/processUpload.js @@ -9,6 +9,7 @@ import { inspectSample, verdictText } from 'utils/upload/sampleInspector'; import UploadStatus from 'utils/upload/UploadStatus'; import loadAndCompressIfNecessary from 'utils/upload/loadAndCompressIfNecessary'; import { inspectFile, Verdict } from 'utils/upload/fileInspector'; +import pushNotificationMessage from 'utils/pushNotificationMessage'; import getFileTypeV2 from 'utils/getFileTypeV2'; @@ -135,6 +136,11 @@ const processUpload = async (filesList, sampleType, samples, experimentId, dispa const filesToUploadForSample = Object.keys(sample.files); + if (!valid) { + pushNotificationMessage('error', `Error uploading sample ${name}: ${validationMessage}`); + return; + } + // Create sample if not exists. try { sample.uuid ??= await dispatch( @@ -142,8 +148,6 @@ const processUpload = async (filesList, sampleType, samples, experimentId, dispa experimentId, name, sampleType, - valid, - validationMessage, filesToUploadForSample, ), ); diff --git a/src/utils/upload/sampleInspector.js b/src/utils/upload/sampleInspector.js index a18fcb8227..2d63cf1b93 100644 --- a/src/utils/upload/sampleInspector.js +++ b/src/utils/upload/sampleInspector.js @@ -11,7 +11,7 @@ const Verdict = { const verdictText = { [Verdict.INVALID_BARCODES_FILE]: 'Barcodes file is invalid', [Verdict.INVALID_FEATURES_FILE]: 'Features file is invalid', - [Verdict.INVALID_TRANSPOSED_MATRIX]: 'Sample files are transposed', + [Verdict.INVALID_TRANSPOSED_MATRIX]: 'Matrix file is transposed', }; const CHUNK_SIZE = 2 ** 18; // 250 kb From 856441f8a38c6f91e82b5a88ebc7b9d4d3c36b56 Mon Sep 17 00:00:00 2001 From: Anugerah Erlaut Date: Mon, 18 Jul 2022 14:28:43 +0100 Subject: [PATCH 09/15] add test to check for notification --- .../utils/upload/processUpload.test.js | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/__test__/utils/upload/processUpload.test.js b/src/__test__/utils/upload/processUpload.test.js index 45620dba45..e9ab870af9 100644 --- a/src/__test__/utils/upload/processUpload.test.js +++ b/src/__test__/utils/upload/processUpload.test.js @@ -14,6 +14,8 @@ import { waitFor } from '@testing-library/dom'; import processUpload from 'utils/upload/processUpload'; import loadAndCompressIfNecessary from 'utils/upload/loadAndCompressIfNecessary'; +import { inspectSample, Verdict } from 'utils/upload/sampleInspector'; +import pushNotificationMessage from 'utils/pushNotificationMessage'; enableFetchMocks(); @@ -122,6 +124,8 @@ jest.mock('axios', () => ({ request: jest.fn(), })); +jest.mock('utils/pushNotificationMessage'); + jest.mock('redux/actions/samples/deleteSamples', () => ({ sendDeleteSamplesRequest: jest.fn(), })); @@ -438,4 +442,24 @@ describe('processUpload', () => { expect(axios.request).not.toHaveBeenCalled(); }); }); + + it('Should not upload sample and show notification if uploaded sample is invalid', async () => { + inspectSample.mockImplementationOnce( + () => ({ valid: false, verdict: [Verdict.INVALID_TRANSPOSED_MATRIX] }), + ); + + await processUpload( + getValidFiles('v2'), + sampleType, + store.getState().samples, + mockExperimentId, + store.dispatch, + ); + + // We do not expect uploads to happen + await waitFor(() => { + expect(pushNotificationMessage).toHaveBeenCalledTimes(1); + expect(axios.request).not.toHaveBeenCalled(); + }); + }); }); From 3eb57c42275109f02aa6b33e00f423ea9fa83ee0 Mon Sep 17 00:00:00 2001 From: Anugerah Erlaut Date: Mon, 18 Jul 2022 14:34:23 +0100 Subject: [PATCH 10/15] revert samples table cells --- .../SamplesTableCells.test.jsx | 2 +- .../data-management/SamplesTableCells.jsx | 22 +++++++++---------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/__test__/components/data-management/SamplesTableCells.test.jsx b/src/__test__/components/data-management/SamplesTableCells.test.jsx index 4b0fc98c19..85d2409a70 100644 --- a/src/__test__/components/data-management/SamplesTableCells.test.jsx +++ b/src/__test__/components/data-management/SamplesTableCells.test.jsx @@ -129,7 +129,7 @@ describe('EditableFieldCell', () => { cellText={mockCellText} dataIndex='mockIndex' rowIdx={1} - onAfterSubmit={() => { }} + onAfterSubmit={() => {}} />); expect(screen.getByText(mockCellText)).toBeInTheDocument(); diff --git a/src/components/data-management/SamplesTableCells.jsx b/src/components/data-management/SamplesTableCells.jsx index 17ac9bbfa2..ab7d369ced 100644 --- a/src/components/data-management/SamplesTableCells.jsx +++ b/src/components/data-management/SamplesTableCells.jsx @@ -2,7 +2,9 @@ import React, { useRef, useState } from 'react'; import { Space, Typography, Progress, Tooltip, Button, } from 'antd'; -import { UploadOutlined } from '@ant-design/icons'; +import { + UploadOutlined, +} from '@ant-design/icons'; import { useDispatch } from 'react-redux'; import PropTypes from 'prop-types'; import styled from 'styled-components'; @@ -165,16 +167,14 @@ const SampleNameCell = (props) => { const { text, record, idx } = cellInfo; const dispatch = useDispatch(); return ( - - - dispatch(updateSample(record.uuid, { name }))} - onDelete={() => dispatch(deleteSamples([record.uuid]))} - /> - - + + dispatch(updateSample(record.uuid, { name }))} + onDelete={() => dispatch(deleteSamples([record.uuid]))} + /> + ); }; SampleNameCell.propTypes = { From d8964c2a2a04bb5582daf27d71e4677709cbdd0b Mon Sep 17 00:00:00 2001 From: Anugerah Erlaut Date: Mon, 18 Jul 2022 14:41:06 +0100 Subject: [PATCH 11/15] minimize mock sample data --- src/__test__/data/mock_files/barcodes.tsv | 3 --- src/__test__/data/mock_files/barcodes.tsv.gz | Bin 84 -> 63 bytes src/__test__/data/mock_files/features.tsv | 7 ------- src/__test__/data/mock_files/features.tsv.gz | Bin 170 -> 102 bytes .../data/mock_files/invalid_barcodes.tsv | 2 -- .../data/mock_files/invalid_barcodes.tsv.gz | Bin 78 -> 57 bytes .../data/mock_files/invalid_features.tsv | 5 ----- .../data/mock_files/invalid_features.tsv.gz | Bin 154 -> 94 bytes src/__test__/data/mock_files/matrix.mtx | 2 +- src/__test__/data/mock_files/matrix.mtx.gz | Bin 136 -> 135 bytes .../data/mock_files/transposed_matrix.mtx | 2 +- .../data/mock_files/transposed_matrix.mtx.gz | Bin 153 -> 152 bytes 12 files changed, 2 insertions(+), 19 deletions(-) diff --git a/src/__test__/data/mock_files/barcodes.tsv b/src/__test__/data/mock_files/barcodes.tsv index 0de05de1e2..4a8c0ef720 100644 --- a/src/__test__/data/mock_files/barcodes.tsv +++ b/src/__test__/data/mock_files/barcodes.tsv @@ -1,5 +1,2 @@ AAACCCATCAAACCTG-1 AAACGAAAGTTGCTGT-1 -AAACGAACACACAGCC-1 -AAACGAATCGTGGGAA-1 -AAACGCTGTTCGGTTA-1 diff --git a/src/__test__/data/mock_files/barcodes.tsv.gz b/src/__test__/data/mock_files/barcodes.tsv.gz index b459d76b2c2a183344db586d3fbea6640067c31e..adced9611d1338df0a7c1415c2af5e5e4b7bb315 100644 GIT binary patch literal 63 zcmb2|=HNJxcr}TEIVrIyIX@+}Sg)kGjG?%sq@=8@jHj@S#n9Vq q5(4sh5C9}$qG9c`TFS^)#6x^1a>$gW#HW4)Go=f^1g^bb00019(jlb) diff --git a/src/__test__/data/mock_files/features.tsv b/src/__test__/data/mock_files/features.tsv index 982f08c8c6..8b9583fed9 100644 --- a/src/__test__/data/mock_files/features.tsv +++ b/src/__test__/data/mock_files/features.tsv @@ -1,10 +1,3 @@ ENSMUSG00000051951 Xkr4 Gene Expression ENSMUSG00000089699 Gm1992 Gene Expression ENSMUSG00000102343 Gm37381 Gene Expression -ENSMUSG00000025900 Rp1 Gene Expression -ENSMUSG00000025902 Sox17 Gene Expression -ENSMUSG00000104328 Gm37323 Gene Expression -ENSMUSG00000033845 Mrpl15 Gene Expression -ENSMUSG00000025903 Lypla1 Gene Expression -ENSMUSG00000104217 Gm37988 Gene Expression -ENSMUSG00000033813 Tcea1 Gene Expression diff --git a/src/__test__/data/mock_files/features.tsv.gz b/src/__test__/data/mock_files/features.tsv.gz index aca846216edb7bacf0389647b40cfae303b3470e..26f6735b64c5cf62f15f44848b2ea321f603d75c 100644 GIT binary patch literal 102 zcmV-s0GaAmbairNb1rmqb^vqr3-%2Sb~gY6Q$tHrL(YioA`?#c z)Vx#$*NTFo)Z*gI{5&pKWOWvnW|o$m?zx7RmPUA08X6cGn-~LC8k-wi7-A>|055rn I>wN$K07Hu>^8f$< literal 170 zcmV;b09F4ViwFoUgVACD17>AmbairNb1rmqb^wizF%AMD5Jl_p6dr+@0hc)dp>Zul z6I){_jO#98oAvghHWmi_^~-zT?LvPHUcF~f48rM_HTXEjwOdk-^L(AgmHXzbY(&7{ zQG}`_Dq_urt<8 diff --git a/src/__test__/data/mock_files/invalid_barcodes.tsv b/src/__test__/data/mock_files/invalid_barcodes.tsv index 9e2abddd25..e53136b4ae 100644 --- a/src/__test__/data/mock_files/invalid_barcodes.tsv +++ b/src/__test__/data/mock_files/invalid_barcodes.tsv @@ -1,3 +1 @@ AAACCCATCAAACCTG-1 -AAACGAAAGTTGCTGT-1 -AAACGAACACACAGCC-1 diff --git a/src/__test__/data/mock_files/invalid_barcodes.tsv.gz b/src/__test__/data/mock_files/invalid_barcodes.tsv.gz index d57ae930fa0c9f6fb1d0ed68b9116d78be513a4b..ed3eb65a44301c32c09cd753745beca4b5d0de68 100644 GIT binary patch delta 23 ecmeaXWS8&e;CPaFbt1baC&S`4TPI-#1_l66kOk-f delta 44 zcmcEYW0&vd;L!heaU#2>oP$oe!b3TsM|>ijjvSWF3wLie4&Y$;E&1|^B?AKk05Ap( A_y7O^ diff --git a/src/__test__/data/mock_files/invalid_features.tsv b/src/__test__/data/mock_files/invalid_features.tsv index 8d0a06861b..e8340c11e1 100644 --- a/src/__test__/data/mock_files/invalid_features.tsv +++ b/src/__test__/data/mock_files/invalid_features.tsv @@ -1,7 +1,2 @@ ENSMUSG00000051951 Xkr4 Gene Expression ENSMUSG00000089699 Gm1992 Gene Expression -ENSMUSG00000102343 Gm37381 Gene Expression -ENSMUSG00000025900 Rp1 Gene Expression -ENSMUSG00000025902 Sox17 Gene Expression -ENSMUSG00000104328 Gm37323 Gene Expression -ENSMUSG00000033845 Mrpl15 Gene Expression diff --git a/src/__test__/data/mock_files/invalid_features.tsv.gz b/src/__test__/data/mock_files/invalid_features.tsv.gz index bb8fef57ae9903766c4c58c59f2844e7e7476314..ffea7ed62fd5009362fcf1bafb18f12ce39d188c 100644 GIT binary patch delta 26 hcmbQm7{@N(&B5_5@#;i&A8rXP#+NO&;z0}y3;=5p2P6Oh delta 86 zcmV-c0IC08ngItNiwFpThtiP;Oh&LyVO)9u005gqJN5to literal 136 zcmV;30C)c%iwFo1_R(Sh18re+a%p%jZFG15P0K+F!axj0;r*QAConrHQ^ig1&@+@^ z(=uuk$OJ2hcUN4wd>_1=M-H#gk^Lrt?H9P1`COGlP&H%~$0{n$_mnpYo;i5BHq!ww qTFk%O^bl{=lm;9Rpk_<#?DEHS>iVwGJM{%;l=K1VlXSd$00022cRo=7 diff --git a/src/__test__/data/mock_files/transposed_matrix.mtx b/src/__test__/data/mock_files/transposed_matrix.mtx index b5f33f2ae3..2c3ebb6487 100644 --- a/src/__test__/data/mock_files/transposed_matrix.mtx +++ b/src/__test__/data/mock_files/transposed_matrix.mtx @@ -1,3 +1,3 @@ %%MatrixMarket matrix coordinate integer general %metadata_json: {"format_version": 2, "software_version": "3.1.0"} -5 10 1357468 +2 3 1357468 diff --git a/src/__test__/data/mock_files/transposed_matrix.mtx.gz b/src/__test__/data/mock_files/transposed_matrix.mtx.gz index de73480c5b3a39a64158d8358c7141a6558a0fa6..ad179a50a2faa9058b8b414fac9fc141fdc30aa0 100644 GIT binary patch delta 122 zcmV-=0EPdV0hj>?ABzYG_F>hL2PbM*itW@Yy+hAXf=$b)O&}9g5bthr0EGXT0hs{@ABzYGiHFjW2PbPMrDLs1@6a=pVAC>c6Uc-rl-^x& Date: Tue, 19 Jul 2022 10:38:58 +0100 Subject: [PATCH 12/15] remove slicing for non-matrix files --- .../utils/upload/processUpload.test.js | 4 +- .../utils/upload/sampleInspector.test.js | 17 ++-- src/utils/upload/processUpload.js | 4 +- src/utils/upload/sampleInspector.js | 83 +++++++++---------- 4 files changed, 52 insertions(+), 56 deletions(-) diff --git a/src/__test__/utils/upload/processUpload.test.js b/src/__test__/utils/upload/processUpload.test.js index e9ab870af9..3b0f243768 100644 --- a/src/__test__/utils/upload/processUpload.test.js +++ b/src/__test__/utils/upload/processUpload.test.js @@ -14,7 +14,7 @@ import { waitFor } from '@testing-library/dom'; import processUpload from 'utils/upload/processUpload'; import loadAndCompressIfNecessary from 'utils/upload/loadAndCompressIfNecessary'; -import { inspectSample, Verdict } from 'utils/upload/sampleInspector'; +import { inspectSample } from 'utils/upload/sampleInspector'; import pushNotificationMessage from 'utils/pushNotificationMessage'; enableFetchMocks(); @@ -445,7 +445,7 @@ describe('processUpload', () => { it('Should not upload sample and show notification if uploaded sample is invalid', async () => { inspectSample.mockImplementationOnce( - () => ({ valid: false, verdict: [Verdict.INVALID_TRANSPOSED_MATRIX] }), + () => ({ valid: false, verdict: ['Some file error'] }), ); await processUpload( diff --git a/src/__test__/utils/upload/sampleInspector.test.js b/src/__test__/utils/upload/sampleInspector.test.js index 3e8b8c1a1c..430f8d4e01 100644 --- a/src/__test__/utils/upload/sampleInspector.test.js +++ b/src/__test__/utils/upload/sampleInspector.test.js @@ -1,4 +1,4 @@ -import { inspectSample, Verdict } from 'utils/upload/sampleInspector'; +import { inspectSample } from 'utils/upload/sampleInspector'; import initialState, { sampleFileTemplate, sampleTemplate } from 'redux/reducers/samples/initialState'; import * as fs from 'fs'; @@ -159,7 +159,9 @@ describe('sampleInspector', () => { }; const result = await inspectSample(mockInvalidBarcodesFile); - expect(result).toEqual({ valid: false, verdict: [Verdict.INVALID_BARCODES_FILE] }); + + expect(result.valid).toEqual(false); + expect(result.verdict[0]).toMatch(/Invalid barcodes.tsv file/i); }); it('Correctly identifies invalid features file', async () => { @@ -176,7 +178,9 @@ describe('sampleInspector', () => { }; const result = await inspectSample(mockInvalidFeaturesFile); - expect(result).toEqual({ valid: false, verdict: [Verdict.INVALID_FEATURES_FILE] }); + + expect(result.valid).toEqual(false); + expect(result.verdict[0]).toMatch(/Invalid features\/genes.tsv file/i); }); it('Correctly identifies transposed matrix file', async () => { @@ -193,9 +197,8 @@ describe('sampleInspector', () => { }; const result = await inspectSample(mockTransposedFile); - expect(result).toEqual({ - valid: false, - verdict: [Verdict.INVALID_TRANSPOSED_MATRIX], - }); + + expect(result.valid).toEqual(false); + expect(result.verdict[0]).toMatch(/Invalid matrix.mtx file/i); }); }); diff --git a/src/utils/upload/processUpload.js b/src/utils/upload/processUpload.js index cf3770bcbc..aac03beba9 100644 --- a/src/utils/upload/processUpload.js +++ b/src/utils/upload/processUpload.js @@ -4,7 +4,7 @@ import _ from 'lodash'; import axios from 'axios'; import { createSample, createSampleFile, updateSampleFileUpload } from 'redux/actions/samples'; -import { inspectSample, verdictText } from 'utils/upload/sampleInspector'; +import { inspectSample } from 'utils/upload/sampleInspector'; import UploadStatus from 'utils/upload/UploadStatus'; import loadAndCompressIfNecessary from 'utils/upload/loadAndCompressIfNecessary'; @@ -132,7 +132,7 @@ const processUpload = async (filesList, sampleType, samples, experimentId, dispa Object.entries(samplesMap).forEach(async ([name, sample]) => { const { valid, verdict } = await inspectSample(sample); - const validationMessage = verdict.map((item) => `${verdictText[item]}`).join('\n'); + const validationMessage = verdict.join('\n'); const filesToUploadForSample = Object.keys(sample.files); diff --git a/src/utils/upload/sampleInspector.js b/src/utils/upload/sampleInspector.js index 2d63cf1b93..ade329abc2 100644 --- a/src/utils/upload/sampleInspector.js +++ b/src/utils/upload/sampleInspector.js @@ -2,20 +2,12 @@ import { DecodeUTF8, Decompress, } from 'fflate'; -const Verdict = { - INVALID_BARCODES_FILE: 'INVALID_SAMPLE_FILES', - INVALID_FEATURES_FILE: 'INVALID_FEATURES_FILE', - INVALID_TRANSPOSED_MATRIX: 'INVALID_TRANSPOSED_MATRIX', +const verdictMessage = { + invalidBarcodesFile: (expected, found) => `Invalid barcodes.tsv file. ${expected} barcodes expected, but ${found} found.`, + invalidFeaturesFile: (expected, found) => `Invalid features/genes.tsv file. ${expected} genes expected, but ${found} found.`, + transposedMatrixFile: () => 'Invalid matrix.mtx file: Matrix is transposed.', }; -const verdictText = { - [Verdict.INVALID_BARCODES_FILE]: 'Barcodes file is invalid', - [Verdict.INVALID_FEATURES_FILE]: 'Features file is invalid', - [Verdict.INVALID_TRANSPOSED_MATRIX]: 'Matrix file is transposed', -}; - -const CHUNK_SIZE = 2 ** 18; // 250 kb - const decode = async (arrBuffer) => { let result = ''; const utfDecode = new DecodeUTF8((data) => { result += data; }); @@ -34,10 +26,11 @@ const decompress = async (arrBuffer) => { const extractSampleSizes = async (matrix) => { const { compressed, fileObject } = matrix; + let header = ''; let matrixHeader = ''; - const fileArrBuffer = await fileObject.slice(0, 500).arrayBuffer(); + const fileArrBuffer = await fileObject.slice(0, 300).arrayBuffer(); matrixHeader = compressed ? await decode(await decompress(fileArrBuffer)) @@ -53,9 +46,10 @@ const extractSampleSizes = async (matrix) => { }; }; -const countLine = async (fileObject, start, compressed) => { - const end = Math.min(start + CHUNK_SIZE, fileObject.size); - const arrBuffer = await fileObject.slice(start, end).arrayBuffer(); +const getNumLines = async (sampleFile) => { + const { compressed, fileObject } = sampleFile; + const arrBuffer = await fileObject.arrayBuffer(); + const fileStr = compressed ? await decode(await decompress(arrBuffer)) : await decode(arrBuffer); @@ -65,42 +59,42 @@ const countLine = async (fileObject, start, compressed) => { return numLines; }; -const getNumLines = async (sampleFile) => { - let pointer = 0; - const counterJobs = []; - - const { compressed, fileObject } = sampleFile; - - while (pointer < fileObject.size) { - counterJobs.push(countLine(fileObject, pointer, compressed)); - pointer += CHUNK_SIZE; - } - - const resultingCounts = await Promise.all(counterJobs); - const numLines = resultingCounts.reduce((count, numLine) => count + numLine, 0); - return numLines; -}; - const validateFileSizes = async (sample) => { const barcodes = sample.files['barcodes.tsv.gz'] || sample.files['barcodes.tsv']; - const features = sample.files['features.tsv.gz'] || sample.files['features.tsv']; + const features = sample.files['features.tsv.gz'] || sample.files['features.tsv'] || sample.files['genes.tsv.gz'] || sample.files['genes.tsv']; const matrix = sample.files['matrix.mtx.gz'] || sample.files['matrix.mtx']; - const { barcodeSize, featuresSize } = await extractSampleSizes(matrix); + const { + barcodeSize: expectedNumBarcodes, + featuresSize: expectedNumFeatures, + } = await extractSampleSizes(matrix); - const numBarcodeLines = await getNumLines(barcodes); - const numFeaturesLines = await getNumLines(features); + const numBarcodeFound = await getNumLines(barcodes); + const numFeaturesFound = await getNumLines(features); const verdict = []; - const valid = numBarcodeLines === barcodeSize && numFeaturesLines === featuresSize; + const valid = numBarcodeFound === expectedNumBarcodes && numFeaturesFound === expectedNumFeatures; + + const isSampleTransposed = numBarcodeFound === expectedNumFeatures + && numFeaturesFound === expectedNumBarcodes; + const isBarcodesInvalid = numBarcodeFound !== expectedNumBarcodes; + const isFeaturesInvalid = numFeaturesFound !== expectedNumFeatures; - const isSampleTransposed = numBarcodeLines === featuresSize && numFeaturesLines === barcodeSize; - const isBarcodesInvalid = numBarcodeLines !== barcodeSize; - const isFeaturesInvalid = numFeaturesLines !== featuresSize; + if (isSampleTransposed) { + verdict.push(verdictMessage.transposedMatrixFile()); + return { valid, verdict }; + } + + if (isBarcodesInvalid) { + verdict.push( + verdictMessage.invalidBarcodesFile(expectedNumBarcodes, numBarcodeFound), + ); + } - if (isSampleTransposed) { verdict.push(Verdict.INVALID_TRANSPOSED_MATRIX); } else { - if (isBarcodesInvalid) verdict.push(Verdict.INVALID_BARCODES_FILE); - if (isFeaturesInvalid) verdict.push(Verdict.INVALID_FEATURES_FILE); + if (isFeaturesInvalid) { + verdict.push( + verdictMessage.invalidFeaturesFile(expectedNumFeatures, numFeaturesFound), + ); } return { valid, verdict }; @@ -131,6 +125,5 @@ const inspectSample = async (sample) => { export { inspectSample, - Verdict, - verdictText, + verdictMessage, }; From 36dd920503284e4dd95fdecbea11fe1d61f01f15 Mon Sep 17 00:00:00 2001 From: Pol Alvarez Date: Wed, 20 Jul 2022 16:59:18 +0200 Subject: [PATCH 13/15] chunking version working --- .../sampleValidator.test.js.snap | 21 +++ .../utils/upload/processUpload.test.js | 22 +-- ...pector.test.js => sampleValidator.test.js} | 92 +++++------- src/utils/upload/processUpload.js | 12 +- src/utils/upload/sampleInspector.js | 129 ---------------- src/utils/upload/sampleValidator.js | 142 ++++++++++++++++++ 6 files changed, 211 insertions(+), 207 deletions(-) create mode 100644 src/__test__/utils/upload/__snapshots__/sampleValidator.test.js.snap rename src/__test__/utils/upload/{sampleInspector.test.js => sampleValidator.test.js} (69%) delete mode 100644 src/utils/upload/sampleInspector.js create mode 100644 src/utils/upload/sampleValidator.js diff --git a/src/__test__/utils/upload/__snapshots__/sampleValidator.test.js.snap b/src/__test__/utils/upload/__snapshots__/sampleValidator.test.js.snap new file mode 100644 index 0000000000..af366ba7bf --- /dev/null +++ b/src/__test__/utils/upload/__snapshots__/sampleValidator.test.js.snap @@ -0,0 +1,21 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`sampleValidator Correctly identifies invalid barcodes file 1`] = ` +Array [ + "Invalid barcodes.tsv file. 2 barcodes expected, but 1 found.", +] +`; + +exports[`sampleValidator Correctly identifies invalid features file 1`] = ` +Array [ + "Invalid features/genes.tsv file. 3 genes expected, but 2 found.", +] +`; + +exports[`sampleValidator Correctly identifies transposed matrix file 1`] = ` +Array [ + "Invalid matrix.mtx file: Matrix is transposed.", + "Invalid barcodes.tsv file. 3 barcodes expected, but 2 found.", + "Invalid features/genes.tsv file. 2 genes expected, but 3 found.", +] +`; diff --git a/src/__test__/utils/upload/processUpload.test.js b/src/__test__/utils/upload/processUpload.test.js index 3b0f243768..875b8f9e96 100644 --- a/src/__test__/utils/upload/processUpload.test.js +++ b/src/__test__/utils/upload/processUpload.test.js @@ -14,7 +14,7 @@ import { waitFor } from '@testing-library/dom'; import processUpload from 'utils/upload/processUpload'; import loadAndCompressIfNecessary from 'utils/upload/loadAndCompressIfNecessary'; -import { inspectSample } from 'utils/upload/sampleInspector'; +import validate from 'utils/upload/sampleValidator'; import pushNotificationMessage from 'utils/pushNotificationMessage'; enableFetchMocks(); @@ -126,18 +126,7 @@ jest.mock('axios', () => ({ jest.mock('utils/pushNotificationMessage'); -jest.mock('redux/actions/samples/deleteSamples', () => ({ - sendDeleteSamplesRequest: jest.fn(), -})); - -jest.mock('utils/upload/sampleInspector', () => { - const actual = jest.requireActual('utils/upload/sampleInspector'); - - return { - ...actual, - inspectSample: jest.fn(() => ({ valid: true, verdict: [] })), - }; -}); +jest.mock('utils/upload/sampleValidator'); let store = null; @@ -153,6 +142,9 @@ describe('processUpload', () => { }); it('Uploads and updates redux correctly when there are no errors with cellranger v3', async () => { + // validate.mockImplementation( + // () => ([]), + // ); const mockAxiosCalls = []; const uploadSuccess = (params) => { mockAxiosCalls.push(params); @@ -444,8 +436,8 @@ describe('processUpload', () => { }); it('Should not upload sample and show notification if uploaded sample is invalid', async () => { - inspectSample.mockImplementationOnce( - () => ({ valid: false, verdict: ['Some file error'] }), + validate.mockImplementationOnce( + () => (['Some file error']), ); await processUpload( diff --git a/src/__test__/utils/upload/sampleInspector.test.js b/src/__test__/utils/upload/sampleValidator.test.js similarity index 69% rename from src/__test__/utils/upload/sampleInspector.test.js rename to src/__test__/utils/upload/sampleValidator.test.js index 430f8d4e01..7b1988eb6d 100644 --- a/src/__test__/utils/upload/sampleInspector.test.js +++ b/src/__test__/utils/upload/sampleValidator.test.js @@ -1,8 +1,10 @@ -import { inspectSample } from 'utils/upload/sampleInspector'; +import validate from 'utils/upload/sampleValidator'; import initialState, { sampleFileTemplate, sampleTemplate } from 'redux/reducers/samples/initialState'; import * as fs from 'fs'; +const _ = require('lodash'); + // A function which returns an object to emulate the Blob class // Required because NodeJS's implementation of Blob modifies the // data, causing it to be irreproducible in the test @@ -44,15 +46,15 @@ jest.mock('fflate', () => { }); const prepareMockFiles = (fileLocations) => { - const result = {}; + const errors = {}; // Read and prepare each file object Object.entries(fileLocations).forEach(([filename, location]) => { const fileUint8Arr = new Uint8Array(fs.readFileSync(location)); - result[filename] = makeBlob(fileUint8Arr); + errors[filename] = makeBlob(fileUint8Arr); }); - return result; + return errors; }; const mockUnzippedFileObjects = prepareMockFiles(mockFileLocations.unziped); @@ -134,71 +136,47 @@ const mockUnzippedSample = { }, }; -describe('sampleInspector', () => { +describe('sampleValidator', () => { it('Correctly pass valid zipped samples', async () => { - const result = await inspectSample(mockZippedSample); - expect(result).toEqual({ valid: true, verdict: [] }); + const errors = await validate(mockZippedSample); + expect(errors).toEqual([]); }); it('Correctly pass valid unzipped samples', async () => { - const result = await inspectSample(mockUnzippedSample); - expect(result).toEqual({ valid: true, verdict: [] }); + const errors = await validate(mockUnzippedSample); + expect(errors).toEqual([]); }); it('Correctly identifies invalid barcodes file', async () => { - const mockInvalidBarcodesFile = { - ...mockZippedSample, - files: { - ...mockZippedSample.files, - 'barcodes.tsv.gz': { - ...mockZippedSample.files['barcodes.tsv.gz'], - fileObject: mockZippedFileObjects['invalid_barcodes.tsv.gz'], - size: mockZippedFileObjects['invalid_barcodes.tsv.gz'].size, - }, - }, - }; - - const result = await inspectSample(mockInvalidBarcodesFile); - - expect(result.valid).toEqual(false); - expect(result.verdict[0]).toMatch(/Invalid barcodes.tsv file/i); + const mockInvalidBarcodesFile = _.cloneDeep(mockZippedSample); + mockInvalidBarcodesFile.files['barcodes.tsv.gz'].fileObject = mockZippedFileObjects['invalid_barcodes.tsv.gz']; + mockInvalidBarcodesFile.files['barcodes.tsv.gz'].size = mockZippedFileObjects['invalid_barcodes.tsv.gz'].size; + + const errors = await validate(mockInvalidBarcodesFile); + + expect(errors.length).toEqual(1); + expect(errors).toMatchSnapshot(); }); it('Correctly identifies invalid features file', async () => { - const mockInvalidFeaturesFile = { - ...mockZippedSample, - files: { - ...mockZippedSample.files, - 'features.tsv.gz': { - ...mockZippedSample.files['features.tsv.gz'], - fileObject: mockZippedFileObjects['invalid_features.tsv.gz'], - size: mockZippedFileObjects['invalid_features.tsv.gz'].size, - }, - }, - }; - - const result = await inspectSample(mockInvalidFeaturesFile); - - expect(result.valid).toEqual(false); - expect(result.verdict[0]).toMatch(/Invalid features\/genes.tsv file/i); + const mockInvalidFeaturesFile = _.cloneDeep(mockZippedSample); + mockInvalidFeaturesFile.files['features.tsv.gz'].fileObject = mockZippedFileObjects['invalid_features.tsv.gz']; + mockInvalidFeaturesFile.files['features.tsv.gz'].size = mockZippedFileObjects['invalid_features.tsv.gz'].size; + + const errors = await validate(mockInvalidFeaturesFile); + + expect(errors.length).toEqual(1); + expect(errors).toMatchSnapshot(); }); it('Correctly identifies transposed matrix file', async () => { - const mockTransposedFile = { - ...mockZippedSample, - files: { - ...mockZippedSample.files, - 'matrix.mtx.gz': { - ...mockZippedSample.files['matrix.mtx.gz'], - fileObject: mockZippedFileObjects['transposed_matrix.mtx.gz'], - size: mockZippedFileObjects['transposed_matrix.mtx.gz'].size, - }, - }, - }; - - const result = await inspectSample(mockTransposedFile); - - expect(result.valid).toEqual(false); - expect(result.verdict[0]).toMatch(/Invalid matrix.mtx file/i); + const mockTransposedFile = _.cloneDeep(mockZippedSample); + mockTransposedFile.files['matrix.mtx.gz'].fileObject = mockZippedFileObjects['transposed_matrix.mtx.gz']; + mockTransposedFile.files['matrix.mtx.gz'].size = mockZippedFileObjects['transposed_matrix.mtx.gz'].size; + + const errors = await validate(mockTransposedFile); + + expect(errors.length).toEqual(3); + expect(errors).toMatchSnapshot(); }); }); diff --git a/src/utils/upload/processUpload.js b/src/utils/upload/processUpload.js index aac03beba9..60b8060d65 100644 --- a/src/utils/upload/processUpload.js +++ b/src/utils/upload/processUpload.js @@ -4,14 +4,14 @@ import _ from 'lodash'; import axios from 'axios'; import { createSample, createSampleFile, updateSampleFileUpload } from 'redux/actions/samples'; -import { inspectSample } from 'utils/upload/sampleInspector'; +import validate from 'utils/upload/sampleValidator'; import UploadStatus from 'utils/upload/UploadStatus'; import loadAndCompressIfNecessary from 'utils/upload/loadAndCompressIfNecessary'; import { inspectFile, Verdict } from 'utils/upload/fileInspector'; -import pushNotificationMessage from 'utils/pushNotificationMessage'; import getFileTypeV2 from 'utils/getFileTypeV2'; +import { message } from 'antd'; const putInS3 = async (loadedFileData, signedUrl, onUploadProgress) => ( await axios.request({ @@ -131,13 +131,13 @@ const processUpload = async (filesList, sampleType, samples, experimentId, dispa }, {}); Object.entries(samplesMap).forEach(async ([name, sample]) => { - const { valid, verdict } = await inspectSample(sample); - const validationMessage = verdict.join('\n'); + const errors = await validate(sample); const filesToUploadForSample = Object.keys(sample.files); - if (!valid) { - pushNotificationMessage('error', `Error uploading sample ${name}: ${validationMessage}`); + if (errors.length > 0) { + const errorMessage = errors.join('\n'); + message.error(`Error uploading sample ${name}.\n${errorMessage}`, 15); return; } diff --git a/src/utils/upload/sampleInspector.js b/src/utils/upload/sampleInspector.js deleted file mode 100644 index ade329abc2..0000000000 --- a/src/utils/upload/sampleInspector.js +++ /dev/null @@ -1,129 +0,0 @@ -import { - DecodeUTF8, Decompress, -} from 'fflate'; - -const verdictMessage = { - invalidBarcodesFile: (expected, found) => `Invalid barcodes.tsv file. ${expected} barcodes expected, but ${found} found.`, - invalidFeaturesFile: (expected, found) => `Invalid features/genes.tsv file. ${expected} genes expected, but ${found} found.`, - transposedMatrixFile: () => 'Invalid matrix.mtx file: Matrix is transposed.', -}; - -const decode = async (arrBuffer) => { - let result = ''; - const utfDecode = new DecodeUTF8((data) => { result += data; }); - utfDecode.push(new Uint8Array(arrBuffer)); - - return result; -}; - -const decompress = async (arrBuffer) => { - let result = ''; - const decompressor = new Decompress((chunk) => { result = chunk; }); - decompressor.push(new Uint8Array(arrBuffer)); - - return result; -}; - -const extractSampleSizes = async (matrix) => { - const { compressed, fileObject } = matrix; - - let header = ''; - let matrixHeader = ''; - - const fileArrBuffer = await fileObject.slice(0, 300).arrayBuffer(); - - matrixHeader = compressed - ? await decode(await decompress(fileArrBuffer)) - : await decode(fileArrBuffer); - - // The matrix header is the first line in the file that splits into 3 - header = matrixHeader.split('\n').find((line) => line.split(' ').length === 3); - - const [featuresSize, barcodeSize] = header.split(' '); - return { - featuresSize: Number.parseInt(featuresSize, 10), - barcodeSize: Number.parseInt(barcodeSize, 10), - }; -}; - -const getNumLines = async (sampleFile) => { - const { compressed, fileObject } = sampleFile; - const arrBuffer = await fileObject.arrayBuffer(); - - const fileStr = compressed - ? await decode(await decompress(arrBuffer)) - : await decode(arrBuffer); - - const numLines = (fileStr.match(/\n|\r\n/g) || []).length; - - return numLines; -}; - -const validateFileSizes = async (sample) => { - const barcodes = sample.files['barcodes.tsv.gz'] || sample.files['barcodes.tsv']; - const features = sample.files['features.tsv.gz'] || sample.files['features.tsv'] || sample.files['genes.tsv.gz'] || sample.files['genes.tsv']; - const matrix = sample.files['matrix.mtx.gz'] || sample.files['matrix.mtx']; - - const { - barcodeSize: expectedNumBarcodes, - featuresSize: expectedNumFeatures, - } = await extractSampleSizes(matrix); - - const numBarcodeFound = await getNumLines(barcodes); - const numFeaturesFound = await getNumLines(features); - - const verdict = []; - const valid = numBarcodeFound === expectedNumBarcodes && numFeaturesFound === expectedNumFeatures; - - const isSampleTransposed = numBarcodeFound === expectedNumFeatures - && numFeaturesFound === expectedNumBarcodes; - const isBarcodesInvalid = numBarcodeFound !== expectedNumBarcodes; - const isFeaturesInvalid = numFeaturesFound !== expectedNumFeatures; - - if (isSampleTransposed) { - verdict.push(verdictMessage.transposedMatrixFile()); - return { valid, verdict }; - } - - if (isBarcodesInvalid) { - verdict.push( - verdictMessage.invalidBarcodesFile(expectedNumBarcodes, numBarcodeFound), - ); - } - - if (isFeaturesInvalid) { - verdict.push( - verdictMessage.invalidFeaturesFile(expectedNumFeatures, numFeaturesFound), - ); - } - - return { valid, verdict }; -}; - -const validationTests = [ - validateFileSizes, -]; - -const inspectSample = async (sample) => { - // The promises return [{ valid: ..., verdict: ... }, ... ] - const validationPromises = validationTests - .map(async (validationFn) => await validationFn(sample)); - - const result = await Promise.all(validationPromises); - - // This transforms it into { valid: ..., verdict: [...] }, - const { valid, verdict } = result.reduce((acc, curr) => ({ - valid: acc.valid && curr.valid, - verdict: [ - ...acc.verdict, - ...curr.verdict, - ], - }), { valid: true, verdict: [] }); - - return { valid, verdict }; -}; - -export { - inspectSample, - verdictMessage, -}; diff --git a/src/utils/upload/sampleValidator.js b/src/utils/upload/sampleValidator.js new file mode 100644 index 0000000000..93c0b4ca12 --- /dev/null +++ b/src/utils/upload/sampleValidator.js @@ -0,0 +1,142 @@ +import { + DecodeUTF8, Decompress, +} from 'fflate'; + +const errorMessages = { + invalidBarcodesFile: (expected, found) => `Invalid barcodes.tsv file. ${expected} barcodes expected, but ${found} found.`, + invalidFeaturesFile: (expected, found) => `Invalid features/genes.tsv file. ${expected} genes expected, but ${found} found.`, + invalidMatrixFile: (expected, found) => `Invalid matrix.tsv file. ${expected} elements expected, but ${found} found.`, + transposedMatrixFile: () => 'Invalid matrix.mtx file: Matrix is transposed.', +}; + +const CHUNK_SIZE = 2 ** 18; // 256 kB + +const decode = async (arrBuffer) => { + let result = ''; + const utfDecode = new DecodeUTF8((data) => { result += data; }); + utfDecode.push(new Uint8Array(arrBuffer)); + + return result; +}; + +const decompress = async (arrBuffer) => { + let result = ''; + const decompressor = new Decompress((chunk) => { result = chunk; }); + decompressor.push(new Uint8Array(arrBuffer)); + + return result; +}; + +const extractSampleSizes = async (matrix) => { + const { compressed, fileObject } = matrix; + + let header = ''; + let matrixHeader = ''; + + const fileArrBuffer = await fileObject.slice(0, 300).arrayBuffer(); + + matrixHeader = compressed + ? await decode(await decompress(fileArrBuffer)) + : await decode(fileArrBuffer); + + // The matrix header is the first line in the file that splits into 3 + header = matrixHeader.split('\n').find((line) => line.split(' ').length === 3); + + console.log('HEADER: ', header); + const [featuresSize, barcodeSize, matrixSize] = header.split(' '); + // const [featuresSize, barcodeSize, matrixSize] = header.split(' '); + return [Number.parseInt(featuresSize, 10), + Number.parseInt(barcodeSize, 10), + Number.parseInt(matrixSize, 10)]; + // matrixSize: Number.parseInt(matrixSize, 10), +}; +const getNumLines = async (sampleFile) => { + const { compressed, fileObject } = sampleFile; + + let numLines = 0; + + // Streaming UTF-8 decoder + const utfDecode = new DecodeUTF8((data) => { + numLines += (data.match(/\n|\r\n/g) || []).length; + }); + + // Streaming Decompression (auto-detect the compression method) + const dcmpStrm = new Decompress((chunk, final) => { + utfDecode.push(chunk, final); + }); + + // if the file is compressed we use the decompressor on top of the utfDecoder + // otherwise, just use the utfDecoder + const decoder = compressed ? dcmpStrm : utfDecode; + + // console.log('decoder object: ', decoder); + console.log('file object size: ', fileObject.size); + let idx = 0; + while (idx + CHUNK_SIZE < fileObject.size) { + console.log('Indices: ', idx, idx + CHUNK_SIZE); + // eslint-disable-next-line no-await-in-loop + const slice = await fileObject.slice(idx, idx + CHUNK_SIZE).arrayBuffer(); + // console.log('slice', slice); + decoder.push(new Uint8Array(slice)); + idx += CHUNK_SIZE; + } + const finalSlice = await fileObject.slice(idx, fileObject.size).arrayBuffer(); + // console.log('final slice', finalSlice); + decoder.push(new Uint8Array(finalSlice), true); + // dcmpStrm.push(fileObject, true); + + console.log('NUM lines: ', numLines); + + return numLines; +}; + +const validateFileSizes = async (sample) => { + const barcodes = sample.files['barcodes.tsv.gz'] || sample.files['barcodes.tsv']; + const features = sample.files['features.tsv.gz'] || sample.files['features.tsv'] || sample.files['genes.tsv.gz'] || sample.files['genes.tsv']; + const matrix = sample.files['matrix.mtx.gz'] || sample.files['matrix.mtx']; + + const [ + expectedNumFeatures, + expectedNumBarcodes, + // expectedMatrixSize, + ] = await extractSampleSizes(matrix); + + const numBarcodesFound = await getNumLines(barcodes); + const numFeaturesFound = await getNumLines(features); + // const numMatrixLinesFound = await getNumLines(matrix); + + const errors = []; + + if (numBarcodesFound === expectedNumFeatures + && numFeaturesFound === expectedNumBarcodes) { + errors.push(errorMessages.transposedMatrixFile()); + } + + if (numBarcodesFound !== expectedNumBarcodes) { + errors.push( + errorMessages.invalidBarcodesFile(expectedNumBarcodes, numBarcodesFound), + ); + } + + if (numFeaturesFound !== expectedNumFeatures) { + errors.push( + errorMessages.invalidFeaturesFile(expectedNumFeatures, numFeaturesFound), + ); + } + + // if ((numMatrixLinesFound - 2) !== expectedMatrixSize) { + // errors.push( + // errorMessages.invalidMatrixFile(expectedMatrixSize, numMatrixLinesFound), + // ); + // } + + return errors; +}; + +const validate = async (sample) => { + const errors = await validateFileSizes(sample); + console.log('errors: ', errors); + return errors; +}; + +export default validate; From 19697bec97dc25bbed1183a6e8d5b9548f4a07cf Mon Sep 17 00:00:00 2001 From: Pol Alvarez Date: Wed, 20 Jul 2022 17:08:44 +0200 Subject: [PATCH 14/15] cleaned up console logs --- .../utils/upload/processUpload.test.js | 3 --- src/utils/pushNotificationMessage.js | 14 ++++++------- src/utils/upload/processUpload.js | 6 +++--- src/utils/upload/sampleValidator.js | 21 ------------------- 4 files changed, 10 insertions(+), 34 deletions(-) diff --git a/src/__test__/utils/upload/processUpload.test.js b/src/__test__/utils/upload/processUpload.test.js index 875b8f9e96..1c3c4e0472 100644 --- a/src/__test__/utils/upload/processUpload.test.js +++ b/src/__test__/utils/upload/processUpload.test.js @@ -142,9 +142,6 @@ describe('processUpload', () => { }); it('Uploads and updates redux correctly when there are no errors with cellranger v3', async () => { - // validate.mockImplementation( - // () => ([]), - // ); const mockAxiosCalls = []; const uploadSuccess = (params) => { mockAxiosCalls.push(params); diff --git a/src/utils/pushNotificationMessage.js b/src/utils/pushNotificationMessage.js index 9927c75019..16a90496c8 100644 --- a/src/utils/pushNotificationMessage.js +++ b/src/utils/pushNotificationMessage.js @@ -1,25 +1,25 @@ import { message } from 'antd'; -const pushNotificationMessage = (type, text) => { +const pushNotificationMessage = (type, text, duration = '') => { switch (type) { case 'success': - message.success(text, 2); + message.success(text, duration || 2); break; case 'error': - message.error(text, 4); + message.error(text, duration || 4); break; case 'info': - message.info(text, 4); + message.info(text, duration || 4); break; case 'warning': case 'warn': - message.warn(text, 4); + message.warn(text, duration || 4); break; case 'loading': - message.loading(text, 2); + message.loading(text, duration || 2); break; default: - message.info(text, 4); + message.info(text, duration || 4); break; } }; diff --git a/src/utils/upload/processUpload.js b/src/utils/upload/processUpload.js index 60b8060d65..db69640209 100644 --- a/src/utils/upload/processUpload.js +++ b/src/utils/upload/processUpload.js @@ -11,7 +11,7 @@ import loadAndCompressIfNecessary from 'utils/upload/loadAndCompressIfNecessary' import { inspectFile, Verdict } from 'utils/upload/fileInspector'; import getFileTypeV2 from 'utils/getFileTypeV2'; -import { message } from 'antd'; +import pushNotificationMessage from 'utils/pushNotificationMessage'; const putInS3 = async (loadedFileData, signedUrl, onUploadProgress) => ( await axios.request({ @@ -135,9 +135,9 @@ const processUpload = async (filesList, sampleType, samples, experimentId, dispa const filesToUploadForSample = Object.keys(sample.files); - if (errors.length > 0) { + if (errors && errors.length > 0) { const errorMessage = errors.join('\n'); - message.error(`Error uploading sample ${name}.\n${errorMessage}`, 15); + pushNotificationMessage('error', `Error uploading sample ${name}.\n${errorMessage}`, 15); return; } diff --git a/src/utils/upload/sampleValidator.js b/src/utils/upload/sampleValidator.js index 93c0b4ca12..d4cc45c0e8 100644 --- a/src/utils/upload/sampleValidator.js +++ b/src/utils/upload/sampleValidator.js @@ -42,13 +42,10 @@ const extractSampleSizes = async (matrix) => { // The matrix header is the first line in the file that splits into 3 header = matrixHeader.split('\n').find((line) => line.split(' ').length === 3); - console.log('HEADER: ', header); const [featuresSize, barcodeSize, matrixSize] = header.split(' '); - // const [featuresSize, barcodeSize, matrixSize] = header.split(' '); return [Number.parseInt(featuresSize, 10), Number.parseInt(barcodeSize, 10), Number.parseInt(matrixSize, 10)]; - // matrixSize: Number.parseInt(matrixSize, 10), }; const getNumLines = async (sampleFile) => { const { compressed, fileObject } = sampleFile; @@ -59,7 +56,6 @@ const getNumLines = async (sampleFile) => { const utfDecode = new DecodeUTF8((data) => { numLines += (data.match(/\n|\r\n/g) || []).length; }); - // Streaming Decompression (auto-detect the compression method) const dcmpStrm = new Decompress((chunk, final) => { utfDecode.push(chunk, final); @@ -69,23 +65,15 @@ const getNumLines = async (sampleFile) => { // otherwise, just use the utfDecoder const decoder = compressed ? dcmpStrm : utfDecode; - // console.log('decoder object: ', decoder); - console.log('file object size: ', fileObject.size); let idx = 0; while (idx + CHUNK_SIZE < fileObject.size) { - console.log('Indices: ', idx, idx + CHUNK_SIZE); // eslint-disable-next-line no-await-in-loop const slice = await fileObject.slice(idx, idx + CHUNK_SIZE).arrayBuffer(); - // console.log('slice', slice); decoder.push(new Uint8Array(slice)); idx += CHUNK_SIZE; } const finalSlice = await fileObject.slice(idx, fileObject.size).arrayBuffer(); - // console.log('final slice', finalSlice); decoder.push(new Uint8Array(finalSlice), true); - // dcmpStrm.push(fileObject, true); - - console.log('NUM lines: ', numLines); return numLines; }; @@ -98,12 +86,10 @@ const validateFileSizes = async (sample) => { const [ expectedNumFeatures, expectedNumBarcodes, - // expectedMatrixSize, ] = await extractSampleSizes(matrix); const numBarcodesFound = await getNumLines(barcodes); const numFeaturesFound = await getNumLines(features); - // const numMatrixLinesFound = await getNumLines(matrix); const errors = []; @@ -124,18 +110,11 @@ const validateFileSizes = async (sample) => { ); } - // if ((numMatrixLinesFound - 2) !== expectedMatrixSize) { - // errors.push( - // errorMessages.invalidMatrixFile(expectedMatrixSize, numMatrixLinesFound), - // ); - // } - return errors; }; const validate = async (sample) => { const errors = await validateFileSizes(sample); - console.log('errors: ', errors); return errors; }; From 0cc705bd36bd0c017aed397681ce9f2b2ea4ac52 Mon Sep 17 00:00:00 2001 From: Pol Alvarez Date: Wed, 20 Jul 2022 19:58:22 +0200 Subject: [PATCH 15/15] removed unused error message --- src/utils/upload/sampleValidator.js | 1 - 1 file changed, 1 deletion(-) diff --git a/src/utils/upload/sampleValidator.js b/src/utils/upload/sampleValidator.js index d4cc45c0e8..4430371d03 100644 --- a/src/utils/upload/sampleValidator.js +++ b/src/utils/upload/sampleValidator.js @@ -5,7 +5,6 @@ import { const errorMessages = { invalidBarcodesFile: (expected, found) => `Invalid barcodes.tsv file. ${expected} barcodes expected, but ${found} found.`, invalidFeaturesFile: (expected, found) => `Invalid features/genes.tsv file. ${expected} genes expected, but ${found} found.`, - invalidMatrixFile: (expected, found) => `Invalid matrix.tsv file. ${expected} elements expected, but ${found} found.`, transposedMatrixFile: () => 'Invalid matrix.mtx file: Matrix is transposed.', };