[ENH] BEP031: Microscopy

bids-validator-web/package.json

@@ -5,6 +5,7 @@
   "main": "index.js",
   "license": "MIT",
   "dependencies": {
+    "@babel/runtime": "^7.16.7",
     "bootstrap": "^4.3.0",
     "bowser": "^1.0.0",
     "next": "^11.1.2",

bids-validator/bids_validator/rules/file_level_rules.json

@@ -523,5 +523,70 @@
     "tokens": {
       "@@@_pet_ext_@@@": ["blood\\.tsv\\.gz", "blood\\.tsv", "blood\\.json"]
     }
+  },
+
+  "microscopy": {
+    "regexp": "^[\\/\\\\](sub-[a-zA-Z0-9]+)[\\/\\\\](?:(ses-[a-zA-Z0-9]+)[\\/\\\\])?micr[\\/\\\\](sub-[a-zA-Z0-9]+)(?:(_ses-[a-zA-Z0-9]+))?(?:_sample-[a-zA-Z0-9]+)(?:_acq-[a-zA-Z0-9]+)?(?:_stain-[a-zA-Z0-9]+)?(?:_run-[0-9]+)?(?:_chunk-[0-9]+)?((@@@_microscopy_type_@@@)(@@@_microscopy_ext_@@@))$",
+    "tokens": {
+      "@@@_microscopy_type_@@@": [
+        "_TEM",
+        "_SEM",
+        "_uCT",
+        "_BF",
+        "_DF",
+        "_PC",
+        "_DIC",
+        "_FLUO",
+        "_CONF",
+        "_PLI",
+        "_CARS",
+        "_2PE",
+        "_MPE",
+        "_SR",
+        "_NLO",
+        "_OCT",
+        "_SPIM"
+      ],
+      "@@@_microscopy_ext_@@@": [
+        ".ome\\.tif",
+        ".ome\\.btf",
+        ".tif",
+        ".png"
+      ]
+    }
+  },
+  "microscopy_photo": {
+    "regexp": "^[\\/\\\\](sub-[a-zA-Z0-9]+)[\\/\\\\](?:(ses-[a-zA-Z0-9]+)[\\/\\\\])?micr[\\/\\\\](sub-[a-zA-Z0-9]+)(?:(_ses-[a-zA-Z0-9]+))?(?:_sample-[a-zA-Z0-9]+)(?:_acq-[a-zA-Z0-9]+)?(@@@_photo_ext_@@@)$",
+    "tokens":{
+      "@@@_photo_ext_@@@": [
+        "_photo\\.jpg",
+        "_photo\\.png",
+        "_photo\\.tif"
+      ]
+    }
+  },
+  "microscopy_json": {
+    "regexp": "^[\\/\\\\](sub-[a-zA-Z0-9]+)[\\/\\\\](?:(ses-[a-zA-Z0-9]+)[\\/\\\\])?micr[\\/\\\\](sub-[a-zA-Z0-9]+)(?:(_ses-[a-zA-Z0-9]+))?(?:_sample-[a-zA-Z0-9]+)?(?:_acq-[a-zA-Z0-9]+)?(?:_stain-[a-zA-Z0-9]+)?(?:_run-[0-9]+)?(?:_chunk-[0-9]+)?(@@@_microscopy_type_@@@)\\.json$",
+    "tokens": {
+      "@@@_microscopy_type_@@@": [
+        "_TEM",
+        "_SEM",
+        "_uCT",
+        "_BF",
+        "_DF",
+        "_PC",
+        "_DIC",
+        "_FLUO",
+        "_CONF",
+        "_PLI",
+        "_CARS",
+        "_2PE",
+        "_MPE",
+        "_SR",
+        "_NLO",
+        "_OCT",
+        "_SPIM"
+      ]
+    }
   }
 }

bids-validator/bids_validator/rules/session_level_rules.json

@@ -201,5 +201,30 @@
         "UNCInfant2V23"
       ]
     }
+  },
+
+  "microscopy_ses": {
+    "regexp": "^[\\/\\\\](sub-[a-zA-Z0-9]+)[\\/\\\\](?:(ses-[a-zA-Z0-9]+)[\\/\\\\])?\\1(_\\2)?(?:_sample-[a-zA-Z0-9]+)?(?:_acq-[a-zA-Z0-9]+)?(?:_stain-[a-zA-Z0-9]+)?(?:_run-[0-9]+_)?(?:_chunk-[0-9]+)?(@@@_microscopy_ses_type_@@@)$",
+    "tokens": {
+      "@@@_microscopy_ses_type_@@@": [
+        "_TEM.json",
+        "_SEM.json",
+        "_uCT.json",
+        "_BF.json",
+        "_DF.json",
+        "_PC.json",
+        "_DIC.json",
+        "_FLUO.json",
+        "_CONF.json",
+        "_PLI.json",
+        "_CARS.json",
+        "_2PE.json",
+        "_MPE.json",
+        "_SR.json",
+        "_NLO.json",
+        "_OCT.json",
+        "_SPIM.json"
+      ]
+    }
   }
 }

bids-validator/bids_validator/rules/top_level_rules.json

@@ -15,7 +15,9 @@
         "phase2.json",
         "fieldmap.json",
         "events.json",
-        "scans.json"
+        "scans.json",
+        "samples.json",
+        "samples.tsv"
       ]
     }
   },
@@ -133,5 +135,30 @@
     "tokens": {
       "@@@_other_top_files_ext_@@@": ["physio\\.json", "stim\\.json"]
     }
+  },
+
+  "microscopy_top": {
+    "regexp": "^[\\/\\\\](?:ses-[a-zA-Z0-9]+_)?(?:_sample-[a-zA-Z0-9]+)?(?:_acq-[a-zA-Z0-9]+)?(?:_stain-[a-zA-Z0-9]+)?(?:_run-[0-9]+)?(?:_chunk-[0-9]+)?(?:@@@_microscopy_top_ext_@@@)$",
+    "tokens": {
+      "@@@_microscopy_top_ext_@@@": [
+        "_TEM\\.json",
+        "_SEM\\.json",
+        "_uCT\\.json",
+        "_BF\\.json",
+        "_DF\\.json",
+        "_PC\\.json",
+        "_DIC\\.json",
+        "_FLUO\\.json",
+        "_CONF\\.json",
+        "_PLI\\.json",
+        "_CARS\\.json",
+        "_2PE\\.json",
+        "_MPE\\.json",
+        "_SR\\.json",
+        "_NLO\\.json",
+        "_OCT\\.json",
+        "_SPIM\\.json"
+      ]
+    }
   }
 }

bids-validator/bin/bids-validator

@@ -8,6 +8,7 @@ function entry(cli) {
 try {
   // Test if there's a development tree to run
   require.resolve('../cli.js')
+  process.env.ESBUILD_MAX_BUFFER = 64 * 1024 * 1024
   // For dev, use esbuild-runner
   require('esbuild-runner/register')
   const { default: cli } = require('../cli.js')

bids-validator/package.json

@@ -58,7 +58,9 @@
     "stream-browserify": "^3.0.0",
     "table": "^5.2.3",
     "yaml": "^1.10.2",
-    "yargs": "^16.2.0"
+    "yargs": "^16.2.0",
+    "exifreader": "^4.1.0",
+    "xml2js": "^0.4.23"
   },
   "devDependencies": {
     "adm-zip": "",

bids-validator/tests/tsv.spec.js

@@ -685,4 +685,30 @@ describe('TSV', function() {
     let issues = validate.TSV.validateContRec([physio_file], {})
     assert(issues.length === 1 && issues[0].code === 133)
   })
+
+  // samples checks -----------------------------------------------------------
+
+  const samplesFile = {
+    name: 'samples.tsv',
+    relativePath: '/samples.tsv',
+  }
+
+  it('should return errors for each missing mandatory header in samples.tsv', () => {
+    const tsv = 'wrong_col\nsome_data\n'
+    validate.TSV.TSV(samplesFile, tsv, [], function(issues) {
+      expect(issues.length).toBe(3)
+      const codes = issues.map(x => x.code)
+      expect(codes.includes(216)).toBe(true)
+      expect(codes.includes(217)).toBe(true)
+      expect(codes.includes(218)).toBe(true)
+    })
+  })
+
+  it('should return an error for invalid sample_type samples.tsv', () => {
+    const tsv = 'sample_type\nbad\n'
+    validate.TSV.TSV(samplesFile, tsv, [], function(issues) {
+      const codes = issues.map(x => x.code)
+      expect(codes.includes(219)).toBe(true)
+    })
+  })
 })

bids-validator/utils/files/index.js

@@ -3,6 +3,8 @@ import FileAPI from './FileAPI'
 
 import newFile from './newFile'
 import readFile from './readFile'
+import readOMEFile from './readOMEFile'
+import readBuffer from './readBuffer'
 import readNiftiHeader from './readNiftiHeader'
 import readDir from './readDir'
 import potentialLocations from './potentialLocations'
@@ -21,6 +23,8 @@ export default {
   newFile,
   readFile,
   readDir,
+  readBuffer,
+  readOMEFile,
   readNiftiHeader,
   generateMergedSidecarDict,
   potentialLocations,

bids-validator/utils/files/readBuffer.js

@@ -0,0 +1,23 @@
+import isNode from '../isNode'
+import fs from 'fs'
+
+const readBuffer = file => {
+  return new Promise((resolve, reject) => {
+    if (isNode) {
+      resolve(fs.readFileSync(file.path))
+    } else {
+      try {
+        const reader = new FileReader()
+        reader.onload = event => {
+          resolve(event.target.result)
+        }
+
+        reader.readAsArrayBuffer(file)
+      } catch (e) {
+        reject(e)
+      }
+    }
+  })
+}
+
+export default readBuffer

bids-validator/utils/files/readFile.js

@@ -25,7 +25,11 @@ const checkEncoding = (file, data, cb) => {
 }
 
 /**
- * Read
+ * readFile
+ * @param {object | File} file - nodeJS fs file or browser File
+ * @param {boolean} annexed - is the file currently annexed?
+ * @param {string} dir - path to directory containing dataset. Only used if
+ *   annexed is true.
  *
  * A helper method for reading file contents.
  * Takes a file object and a callback and calls
@@ -39,12 +43,12 @@ const checkEncoding = (file, data, cb) => {
 function readFile(file, annexed, dir) {
   return new Promise((resolve, reject) => {
     if (isNode) {
-      testFile(file, annexed, dir, function (issue, stats, remoteBuffer) {
+      testFile(file, annexed, dir, function(issue, stats, remoteBuffer) {
         if (issue) {
           return reject(issue)
         }
         if (!remoteBuffer) {
-          fs.readFile(file.path, function (err, data) {
+          fs.readFile(file.path, function(err, data) {
             if (err) {
               return reject(err)
             }

bids-validator/utils/files/readOMEFile.js

@@ -0,0 +1,17 @@
+import ExifReader from 'exifreader'
+const xml2js = require('xml2js')
+
+const readOMEFile = buffer => {
+  let tags = ExifReader.load(buffer)
+  let xml = tags['ImageDescription']['description']
+  return new Promise((resolve, reject) => {
+    xml2js
+      .parseStringPromise(xml)
+      .then(result => {
+        resolve(result)
+      })
+      .catch(error => reject(error))
+  })
+}
+
+export default readOMEFile

bids-validator/utils/issues/list.js

@@ -1042,4 +1042,80 @@ export default {
     reason:
       'The recommended file /README is very small. Please consider expanding it with additional information about the dataset.',
   },
+  214: {
+    key: 'SAMPLES_TSV_MISSING',
+    severity: 'error',
+    reason:
+      'The compulsory file /samples.tsv is missing. See Section 03 (Modality agnostic files) of the BIDS specification.',
+  },
+  215: {
+    key: 'SAMPLE_ID_PATTERN',
+    severity: 'error',
+    reason:
+      'sample_id column labels must consist of the pattern "sample-<sample_id>".',
+  },
+  216: {
+    key: 'SAMPLE_ID_COLUMN',
+    severity: 'error',
+    reason: "Samples .tsv files must have a 'sample_id' column.",
+  },
+  217: {
+    key: 'PARTICIPANT_ID_COLUMN',
+    severity: 'error',
+    reason: "Samples .tsv files must have a 'participant_id' column.",
+  },
+  218: {
+    key: 'SAMPLE_TYPE_COLUMN',
+    severity: 'error',
+    reason: "Samples .tsv files must have a 'sample_type' column.",
+  },
+  219: {
+    key: 'SAMPLE_TYPE_VALUE',
+    severity: 'error',
+    reason:
+      'sample_type MUST consist of one of the following values: cell line, in vitro differentiated cells, primary cell, cell-free sample, cloning host, tissue, whole organisms, organoid or technical sample.',
+  },
+  220: {
+    key: 'SAMPLE_ID_DUPLICATE',
+    severity: 'error',
+    reason:
+      'Each sample from a same subject MUST be described by one and only one row.',
+  },
+  221: {
+    key: 'PIXEL_SIZE_INCONSISTENT',
+    severity: 'error',
+    reason:
+      'PixelSize need to be consistent with PhysicalSizeX, PhysicalSizeY and PhysicalSizeZ OME metadata fields',
+  },
+  222: {
+    key: 'INVALID_PIXEL_SIZE_UNIT',
+    severity: 'warning',
+    reason: 'PixelSize consistency is only validated for "mm", "µm" and "nm".',
+  },
+  223: {
+    key: 'CHUNK_TRANSFORMATION_MATRIX_MISSING',
+    severity: 'warning',
+    reason:
+      "It is recommended to define 'ChunkTransformationMatrix' for this file.",
+  },
+  224: {
+    key: 'OPTIONAL_FIELD_INCONSISTENT',
+    severity: 'error',
+    reason: 'Optional JSON field is not consistent with the OME-TIFF metadata',
+  },
+  225: {
+    key: 'NO_VALID_JSON',
+    severity: 'error',
+    reason: 'No valid JSON file found for this file',
+  },
+  226: {
+    key: 'UNSUPPORTED_BIG_TIFF',
+    severity: 'warning',
+    reason: 'Metadata consistency check skipped for BigTiff OME-TIFF file',
+  },
+  227: {
+    key: 'INCONSISTENT_TIFF_EXTENSION',
+    severity: 'error',
+    reason: 'Inconsistent TIFF file type and extension',
+  },
 }

bids-validator/utils/summary/collectModalities.js

@@ -7,6 +7,7 @@ export const collectModalities = filenames => {
     MEG: 0,
     EEG: 0,
     iEEG: 0,
+    Microscopy: 0,
   }
   const secondary = {
     MRI_Diffusion: 0,
@@ -56,6 +57,9 @@ export const collectModalities = filenames => {
     if (type.file.isIEEG(path)) {
       modalities.iEEG++
     }
+    if (type.file.isMicroscopy(path)) {
+      modalities.Microscopy++
+    }
   }
   // Order by matching file count
   const nonZero = Object.keys(modalities).filter(a => modalities[a] !== 0)