[BIOMAGE-1774] - Check sample file sizes

src/__test__/data/mock_files/barcodes.tsv

@@ -0,0 +1,2 @@
+AAACCCATCAAACCTG-1
+AAACGAAAGTTGCTGT-1

src/__test__/data/mock_files/features.tsv

@@ -0,0 +1,3 @@
+ENSMUSG00000051951	Xkr4	Gene Expression
+ENSMUSG00000089699	Gm1992	Gene Expression
+ENSMUSG00000102343	Gm37381	Gene Expression

src/__test__/data/mock_files/invalid_barcodes.tsv

@@ -0,0 +1 @@
+AAACCCATCAAACCTG-1

src/__test__/data/mock_files/invalid_features.tsv

@@ -0,0 +1,2 @@
+ENSMUSG00000051951	Xkr4	Gene Expression
+ENSMUSG00000089699	Gm1992	Gene Expression

src/__test__/data/mock_files/matrix.mtx

@@ -0,0 +1,3 @@
+%%MatrixMarket matrix coordinate integer general
+%metadata_json: {"format_version": 2, "software_version": "3.1.0"}
+3 2 0

src/__test__/data/mock_files/transposed_matrix.mtx

@@ -0,0 +1,3 @@
+%%MatrixMarket matrix coordinate integer general
+%metadata_json: {"format_version": 2, "software_version": "3.1.0"}
+2 3 1357468

src/__test__/utils/upload/__snapshots__/sampleValidator.test.js.snap

@@ -0,0 +1,21 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`sampleValidator Correctly identifies invalid barcodes file 1`] = `
+Array [
+  "Invalid barcodes.tsv file. 2 barcodes expected, but 1 found.",
+]
+`;
+
+exports[`sampleValidator Correctly identifies invalid features file 1`] = `
+Array [
+  "Invalid features/genes.tsv file. 3 genes expected, but 2 found.",
+]
+`;
+
+exports[`sampleValidator Correctly identifies transposed matrix file 1`] = `
+Array [
+  "Invalid matrix.mtx file: Matrix is transposed.",
+  "Invalid barcodes.tsv file. 3 barcodes expected, but 2 found.",
+  "Invalid features/genes.tsv file. 2 genes expected, but 3 found.",
+]
+`;

src/__test__/utils/upload/processUpload.test.js

@@ -14,6 +14,8 @@ import { waitFor } from '@testing-library/dom';
 import processUpload from 'utils/upload/processUpload';
 
 import loadAndCompressIfNecessary from 'utils/upload/loadAndCompressIfNecessary';
+import validate from 'utils/upload/sampleValidator';
+import pushNotificationMessage from 'utils/pushNotificationMessage';
 
 enableFetchMocks();
 
@@ -122,9 +124,9 @@ jest.mock('axios', () => ({
   request: jest.fn(),
 }));
 
-jest.mock('redux/actions/samples/deleteSamples', () => ({
-  sendDeleteSamplesRequest: jest.fn(),
-}));
+jest.mock('utils/pushNotificationMessage');
+
+jest.mock('utils/upload/sampleValidator');
 
 let store = null;
 
@@ -429,4 +431,24 @@ describe('processUpload', () => {
       expect(axios.request).not.toHaveBeenCalled();
     });
   });
+
+  it('Should not upload sample and show notification if uploaded sample is invalid', async () => {
+    validate.mockImplementationOnce(
+      () => (['Some file error']),
+    );
+
+    await processUpload(
+      getValidFiles('v2'),
+      sampleType,
+      store.getState().samples,
+      mockExperimentId,
+      store.dispatch,
+    );
+
+    // We do not expect uploads to happen
+    await waitFor(() => {
+      expect(pushNotificationMessage).toHaveBeenCalledTimes(1);
+      expect(axios.request).not.toHaveBeenCalled();
+    });
+  });
 });

src/__test__/utils/upload/sampleValidator.test.js

@@ -0,0 +1,182 @@
+import validate from 'utils/upload/sampleValidator';
+import initialState, { sampleFileTemplate, sampleTemplate } from 'redux/reducers/samples/initialState';
+
+import * as fs from 'fs';
+
+const _ = require('lodash');
+
+// A function which returns an object to emulate the Blob class
+// Required because NodeJS's implementation of Blob modifies the
+// data, causing it to be irreproducible in the test
+const makeBlob = (data) => ({
+  data,
+  size: data.length,
+  slice(start, end) { return makeBlob(data.slice(start, end)); },
+  arrayBuffer() { return new Promise((resolve) => resolve(data.buffer)); },
+});
+
+const mockFileLocations = {
+  unziped: {
+    'barcodes.tsv': 'src/__test__/data/mock_files/barcodes.tsv',
+    'features.tsv': 'src/__test__/data/mock_files/features.tsv',
+    'matrix.mtx': 'src/__test__/data/mock_files/matrix.mtx',
+    'invalid_barcodes.tsv': 'src/__test__/data/mock_files/invalid_barcodes.tsv',
+    'invalid_features.tsv': 'src/__test__/data/mock_files/invalid_features.tsv',
+    'transposed_matrix.mtx': 'src/__test__/data/mock_files/transposed_matrix.mtx',
+  },
+  zipped: {
+    'barcodes.tsv.gz': 'src/__test__/data/mock_files/barcodes.tsv.gz',
+    'features.tsv.gz': 'src/__test__/data/mock_files/features.tsv.gz',
+    'matrix.mtx.gz': 'src/__test__/data/mock_files/matrix.mtx.gz',
+    'invalid_barcodes.tsv.gz': 'src/__test__/data/mock_files/invalid_barcodes.tsv.gz',
+    'invalid_features.tsv.gz': 'src/__test__/data/mock_files/invalid_features.tsv.gz',
+    'transposed_matrix.mtx.gz': 'src/__test__/data/mock_files/transposed_matrix.mtx.gz',
+  },
+};
+
+// Manually resolve fflate import for Jest the module definition for fflate
+// because it does not correctly resolve to the intended module
+jest.mock('fflate', () => {
+  const realModule = jest.requireActual('../../../../node_modules/fflate/umd/index.js');
+
+  return {
+    _esModule: true,
+    ...realModule,
+  };
+});
+
+const prepareMockFiles = (fileLocations) => {
+  const errors = {};
+
+  // Read and prepare each file object
+  Object.entries(fileLocations).forEach(([filename, location]) => {
+    const fileUint8Arr = new Uint8Array(fs.readFileSync(location));
+    errors[filename] = makeBlob(fileUint8Arr);
+  });
+
+  return errors;
+};
+
+const mockUnzippedFileObjects = prepareMockFiles(mockFileLocations.unziped);
+const mockZippedFileObjects = prepareMockFiles(mockFileLocations.zipped);
+
+jest.mock('utils/upload/readFileToBuffer');
+
+const mockZippedSample = {
+  ...sampleTemplate,
+  ...initialState,
+  name: 'mockZippedSample',
+  fileNames: [
+    'features.tsv.gz',
+    'barcodes.tsv.gz',
+    'matrix.mtx.gz',
+  ],
+  files: {
+    'features.tsv.gz': {
+      ...sampleFileTemplate,
+      name: 'features.tsv.gz',
+      fileObject: mockZippedFileObjects['features.tsv.gz'],
+      size: mockZippedFileObjects['features.tsv.gz'].size,
+      path: '/transposed/features.tsv.gz',
+      compressed: true,
+    },
+    'barcodes.tsv.gz': {
+      ...sampleFileTemplate,
+      name: 'barcodes.tsv.gz',
+      fileObject: mockZippedFileObjects['barcodes.tsv.gz'],
+      size: mockZippedFileObjects['barcodes.tsv.gz'].size,
+      path: '/transposed/barcodes.tsv.gz',
+      compressed: true,
+    },
+    'matrix.mtx.gz': {
+      ...sampleFileTemplate,
+      name: 'matrix.mtx.gz',
+      fileObject: mockZippedFileObjects['matrix.mtx.gz'],
+      size: mockZippedFileObjects['matrix.mtx.gz'].size,
+      path: '/transposed/matrix.mtx.gz',
+      compressed: true,
+    },
+  },
+};
+
+const mockUnzippedSample = {
+  ...sampleTemplate,
+  ...initialState,
+  name: 'mockUnzippedSample',
+  fileNames: [
+    'features.tsv',
+    'barcodes.tsv',
+    'matrix.mtx',
+  ],
+  files: {
+    'features.tsv': {
+      ...sampleFileTemplate,
+      name: 'features.tsv',
+      fileObject: mockUnzippedFileObjects['features.tsv'],
+      size: mockUnzippedFileObjects['features.tsv'].size,
+      path: '/transposed/features.tsv',
+      compressed: false,
+    },
+    'barcodes.tsv': {
+      ...sampleFileTemplate,
+      name: 'barcodes.tsv',
+      fileObject: mockUnzippedFileObjects['barcodes.tsv'],
+      size: mockUnzippedFileObjects['barcodes.tsv'].size,
+      path: '/transposed/barcodes.tsv',
+      compressed: false,
+    },
+    'matrix.mtx': {
+      ...sampleFileTemplate,
+      name: 'matrix.mtx',
+      fileObject: mockUnzippedFileObjects['matrix.mtx'],
+      size: mockUnzippedFileObjects['matrix.mtx'].size,
+      path: '/transposed/matrix.mtx',
+      compressed: false,
+    },
+  },
+};
+
+describe('sampleValidator', () => {
+  it('Correctly pass valid zipped samples', async () => {
+    const errors = await validate(mockZippedSample);
+    expect(errors).toEqual([]);
+  });
+
+  it('Correctly pass valid unzipped samples', async () => {
+    const errors = await validate(mockUnzippedSample);
+    expect(errors).toEqual([]);
+  });
+
+  it('Correctly identifies invalid barcodes file', async () => {
+    const mockInvalidBarcodesFile = _.cloneDeep(mockZippedSample);
+    mockInvalidBarcodesFile.files['barcodes.tsv.gz'].fileObject = mockZippedFileObjects['invalid_barcodes.tsv.gz'];
+    mockInvalidBarcodesFile.files['barcodes.tsv.gz'].size = mockZippedFileObjects['invalid_barcodes.tsv.gz'].size;
+
+    const errors = await validate(mockInvalidBarcodesFile);
+
+    expect(errors.length).toEqual(1);
+    expect(errors).toMatchSnapshot();
+  });
+
+  it('Correctly identifies invalid features file', async () => {
+    const mockInvalidFeaturesFile = _.cloneDeep(mockZippedSample);
+    mockInvalidFeaturesFile.files['features.tsv.gz'].fileObject = mockZippedFileObjects['invalid_features.tsv.gz'];
+    mockInvalidFeaturesFile.files['features.tsv.gz'].size = mockZippedFileObjects['invalid_features.tsv.gz'].size;
+
+    const errors = await validate(mockInvalidFeaturesFile);
+
+    expect(errors.length).toEqual(1);
+    expect(errors).toMatchSnapshot();
+  });
+
+  it('Correctly identifies transposed matrix file', async () => {
+    const mockTransposedFile = _.cloneDeep(mockZippedSample);
+    mockTransposedFile.files['matrix.mtx.gz'].fileObject = mockZippedFileObjects['transposed_matrix.mtx.gz'];
+    mockTransposedFile.files['matrix.mtx.gz'].size = mockZippedFileObjects['transposed_matrix.mtx.gz'].size;
+
+    const errors = await validate(mockTransposedFile);
+
+    expect(errors.length).toEqual(3);
+    expect(errors).toMatchSnapshot();
+  });
+});

src/redux/actions/samples/loadSamples.js

@@ -75,8 +75,6 @@ const loadSamples = (experimentId) => async (dispatch) => {
 
     const samples = toApiV1(data, experimentId);
 
-    // throwIfRequestFailed(response, data, endUserMessages.ERROR_FETCHING_SAMPLES);
-
     dispatch({
       type: SAMPLES_LOADED,
       payload: {

src/utils/pushNotificationMessage.js

@@ -1,25 +1,25 @@
 import { message } from 'antd';
 
-const pushNotificationMessage = (type, text) => {
+const pushNotificationMessage = (type, text, duration = '') => {
   switch (type) {
     case 'success':
-      message.success(text, 2);
+      message.success(text, duration || 2);
       break;
     case 'error':
-      message.error(text, 4);
+      message.error(text, duration || 4);
       break;
     case 'info':
-      message.info(text, 4);
+      message.info(text, duration || 4);
       break;
     case 'warning':
     case 'warn':
-      message.warn(text, 4);
+      message.warn(text, duration || 4);
       break;
     case 'loading':
-      message.loading(text, 2);
+      message.loading(text, duration || 2);
       break;
     default:
-      message.info(text, 4);
+      message.info(text, duration || 4);
       break;
   }
 };

src/utils/upload/processUpload.js

@@ -4,12 +4,14 @@ import _ from 'lodash';
 import axios from 'axios';
 
 import { createSample, createSampleFile, updateSampleFileUpload } from 'redux/actions/samples';
+import validate from 'utils/upload/sampleValidator';
 
 import UploadStatus from 'utils/upload/UploadStatus';
 import loadAndCompressIfNecessary from 'utils/upload/loadAndCompressIfNecessary';
 import { inspectFile, Verdict } from 'utils/upload/fileInspector';
 
 import getFileTypeV2 from 'utils/getFileTypeV2';
+import pushNotificationMessage from 'utils/pushNotificationMessage';
 
 const putInS3 = async (loadedFileData, signedUrl, onUploadProgress) => (
   await axios.request({
@@ -129,12 +131,25 @@ const processUpload = async (filesList, sampleType, samples, experimentId, dispa
   }, {});
 
   Object.entries(samplesMap).forEach(async ([name, sample]) => {
+    const errors = await validate(sample);
+
     const filesToUploadForSample = Object.keys(sample.files);
 
+    if (errors && errors.length > 0) {
+      const errorMessage = errors.join('\n');
+      pushNotificationMessage('error', `Error uploading sample ${name}.\n${errorMessage}`, 15);
+      return;
+    }
+
     // Create sample if not exists.
     try {
       sample.uuid ??= await dispatch(
-        createSample(experimentId, name, sampleType, filesToUploadForSample),
+        createSample(
+          experimentId,
+          name,
+          sampleType,
+          filesToUploadForSample,
+        ),
       );
     } catch (e) {
       // If sample creation fails, sample should not be created