Merge pull request hms-dbmi-cellenics#488 from biomage-org/cell-level…

…-meta

[Cell-level metadata] Update the Add metadata select

src/api.v2/controllers/cellLevelMetaController.js

@@ -0,0 +1,48 @@
+const { v4: uuidv4 } = require('uuid');
+const _ = require('lodash');
+const bucketNames = require('../../config/bucketNames');
+const sqlClient = require('../../sql/sqlClient');
+const CellLevelMeta = require('../model/CellLevelMeta');
+const CellLevelMetaToExperiment = require('../model/CellLevelMetaToExperiment');
+const { getFileUploadUrls } = require('../helpers/s3/signedUrl');
+const OK = require('../../utils/responses/OK');
+
+const uploadCellLevelMetadata = async (req, res) => {
+  const { experimentId } = req.params;
+  const { name, size } = req.body;
+
+  const cellLevelMetaKey = uuidv4();
+  const bucketName = bucketNames.CELL_METADATA;
+  const newCellLevelMetaFile = {
+    id: cellLevelMetaKey,
+    name,
+    upload_status: 'uploading',
+  };
+  const cellLevelMetaToExperimentMap = {
+    experiment_id: experimentId,
+    cell_metadata_file_id: cellLevelMetaKey,
+  };
+
+  let uploadUrlParams;
+  await sqlClient.get().transaction(async (trx) => {
+    await new CellLevelMeta(trx).create(newCellLevelMetaFile);
+    await new CellLevelMetaToExperiment(trx).create(cellLevelMetaToExperimentMap);
+    uploadUrlParams = await getFileUploadUrls(cellLevelMetaKey, {}, size, bucketName);
+    uploadUrlParams = { ...uploadUrlParams, fileId: cellLevelMetaKey };
+  });
+
+  res.json({ data: uploadUrlParams });
+};
+
+
+const updateCellLevelMetadata = async (req, res) => {
+  const { params: { experimentId }, body } = req;
+  const snakeCasedKeysToPatch = _.mapKeys(body, (_value, key) => _.snakeCase(key));
+  const { cellMetadataFileId } = await new CellLevelMetaToExperiment().find({ experiment_id: experimentId }).first();
+  await new CellLevelMeta().updateById(cellMetadataFileId, snakeCasedKeysToPatch);
+  res.json(OK());
+};
+module.exports = {
+  uploadCellLevelMetadata,
+  updateCellLevelMetadata,
+};

src/api.v2/controllers/experimentController.js

@@ -69,7 +69,6 @@ const getExampleExperiments = async (req, res) => {
 const getExperiment = async (req, res) => {
   const { params: { experimentId } } = req;
   logger.log(`Getting experiment ${experimentId}`);
-
   const data = await new Experiment().getExperimentData(experimentId);
 
   logger.log(`Finished getting experiment ${experimentId}`);
@@ -146,7 +145,6 @@ const getProcessingConfig = async (req, res) => {
   logger.log('Getting processing config for experiment ', experimentId);
 
   const result = await new Experiment().getProcessingConfig(experimentId);
-
   logger.log('Finished getting processing config for experiment ', experimentId);
   res.json(result);
 };

src/api.v2/controllers/s3Upload.js

@@ -0,0 +1,30 @@
+const { completeMultipartUpload } = require('../helpers/s3/signedUrl');
+
+const bucketNames = require('../../config/bucketNames');
+const getLogger = require('../../utils/getLogger');
+const { OK, NotFoundError } = require('../../utils/responses');
+
+const logger = getLogger();
+const completeMultipartUploads = async (req, res) => {
+  const {
+    fileId, uploadId, parts, type,
+  } = req.body;
+  let bucketName;
+  if (type === 'sample') {
+    bucketName = bucketNames.SAMPLE_FILES;
+  } else if (type === 'cellLevel') {
+    bucketName = bucketNames.CELL_METADATA;
+  } else {
+    throw new NotFoundError('Invalid bucket specified');
+  }
+  logger.log(`completing multipart upload for file ${fileId}`);
+
+  await completeMultipartUpload(fileId, parts, uploadId, bucketName);
+
+  logger.log(`completed multipart upload for file ${fileId}`);
+  res.json(OK());
+};
+
+module.exports = {
+  completeMultipartUploads,
+};

src/api.v2/controllers/sampleFileController.js

@@ -2,8 +2,9 @@ const sqlClient = require('../../sql/sqlClient');
 
 const Sample = require('../model/Sample');
 const SampleFile = require('../model/SampleFile');
+const bucketNames = require('../../config/bucketNames');
 
-const { getSampleFileUploadUrls, getSampleFileDownloadUrl, completeMultipartUpload } = require('../helpers/s3/signedUrl');
+const { getFileUploadUrls, getSampleFileDownloadUrl } = require('../helpers/s3/signedUrl');
 const { OK } = require('../../utils/responses');
 const getLogger = require('../../utils/getLogger');
 
@@ -31,7 +32,7 @@ const createFile = async (req, res) => {
     await new Sample(trx).setNewFile(sampleId, sampleFileId, sampleFileType);
 
     logger.log(`Getting multipart upload urls for ${experimentId}, sample ${sampleId}, sampleFileType ${sampleFileType}`);
-    uploadUrlParams = await getSampleFileUploadUrls(sampleFileId, metadata, size);
+    uploadUrlParams = await getFileUploadUrls(sampleFileId, metadata, size, bucketNames.SAMPLE_FILES);
   });
 
 
@@ -52,19 +53,6 @@ const patchFile = async (req, res) => {
   res.json(OK());
 };
 
-const completeMultipart = async (req, res) => {
-  const {
-    body: { sampleFileId, parts, uploadId },
-  } = req;
-
-  logger.log(`completing multipart upload for sampleFileId ${sampleFileId}`);
-
-  completeMultipartUpload(sampleFileId, parts, uploadId);
-
-  logger.log(`completed multipart upload for sampleFileId ${sampleFileId}`);
-  res.json(OK());
-};
-
 const getS3DownloadUrl = async (req, res) => {
   const { experimentId, sampleId, sampleFileType } = req.params;
 
@@ -78,5 +66,5 @@ const getS3DownloadUrl = async (req, res) => {
 };
 
 module.exports = {
-  createFile, patchFile, getS3DownloadUrl, completeMultipart,
+  createFile, patchFile, getS3DownloadUrl,
 };

src/api.v2/helpers/s3/signedUrl.js

@@ -64,15 +64,14 @@ const createMultipartUpload = async (params, size) => {
 };
 
 
-const completeMultipartUpload = async (sampleFileId, parts, uploadId) => {
+const completeMultipartUpload = async (key, parts, uploadId, bucketName) => {
   const params = {
-    Bucket: bucketNames.SAMPLE_FILES,
-    Key: `${sampleFileId}`,
+    Bucket: bucketName,
+    Key: key,
     UploadId: uploadId,
     MultipartUpload: { Parts: parts },
   };
 
-
   const S3Config = {
     apiVersion: '2006-03-01',
     signatureVersion: 'v4',
@@ -83,10 +82,10 @@ const completeMultipartUpload = async (sampleFileId, parts, uploadId) => {
   await s3.completeMultipartUpload(params).promise();
 };
 
-const getSampleFileUploadUrls = async (sampleFileId, metadata, size) => {
+const getFileUploadUrls = async (key, metadata, size, bucketName) => {
   const params = {
-    Bucket: bucketNames.SAMPLE_FILES,
-    Key: sampleFileId,
+    Bucket: bucketName,
+    Key: key,
     // 1 hour timeout of upload link
     Expires: 3600,
   };
@@ -130,7 +129,7 @@ const getSampleFileDownloadUrl = async (experimentId, sampleId, fileType) => {
 };
 
 module.exports = {
-  getSampleFileUploadUrls,
+  getFileUploadUrls,
   getSampleFileDownloadUrl,
   getSignedUrl,
   createMultipartUpload,

src/api.v2/model/CellLevelMeta.js

@@ -0,0 +1,32 @@
+const BasicModel = require('./BasicModel');
+const sqlClient = require('../../sql/sqlClient');
+const tableNames = require('./tableNames');
+
+const fields = [
+  'id',
+  'name',
+  'upload_status',
+  'created_at',
+];
+
+class CellLevelMeta extends BasicModel {
+  constructor(sql = sqlClient.get()) {
+    super(sql, tableNames.CELL_LEVEL, fields);
+  }
+
+  async getMetadataByExperimentIds(experimentIds) {
+    const result = await this.sql
+      .select([...fields, `${tableNames.CELL_LEVEL_TO_EXPERIMENT_MAP}.experiment_id`])
+      .from(tableNames.CELL_LEVEL_TO_EXPERIMENT_MAP)
+      .leftJoin(
+        tableNames.CELL_LEVEL,
+        `${tableNames.CELL_LEVEL_TO_EXPERIMENT_MAP}.cell_metadata_file_id`,
+        `${tableNames.CELL_LEVEL}.id`,
+      )
+      .whereIn(`${tableNames.CELL_LEVEL_TO_EXPERIMENT_MAP}.experiment_id`, experimentIds);
+
+    return result;
+  }
+}
+
+module.exports = CellLevelMeta;

src/api.v2/model/CellLevelMetaToExperiment.js

@@ -0,0 +1,16 @@
+const BasicModel = require('./BasicModel');
+const sqlClient = require('../../sql/sqlClient');
+const tableNames = require('./tableNames');
+
+const fields = [
+  'experiment_id',
+  'cell_metadata_file_id',
+];
+
+class CellLevelMetaToExperiment extends BasicModel {
+  constructor(sql = sqlClient.get()) {
+    super(sql, tableNames.CELL_LEVEL_TO_EXPERIMENT_MAP, fields);
+  }
+}
+
+module.exports = CellLevelMetaToExperiment;

src/api.v2/model/Experiment.js

@@ -1,10 +1,11 @@
+/* eslint-disable no-param-reassign */
 const _ = require('lodash');
 const { v4: uuidv4 } = require('uuid');
 
 const BasicModel = require('./BasicModel');
 const sqlClient = require('../../sql/sqlClient');
 const { collapseKeyIntoArray, replaceNullsWithObject } = require('../../sql/helpers');
-
+const CellLevelMeta = require('./CellLevelMeta');
 const { NotFoundError, BadRequestError } = require('../../utils/responses');
 const tableNames = require('./tableNames');
 const config = require('../../config');
@@ -68,17 +69,31 @@ class Experiment extends BasicModel {
       .leftJoin(`${tableNames.EXPERIMENT_PARENT} as p`, 'e.id', 'p.experiment_id')
       .as('mainQuery');
 
-    const result = await collapseKeyIntoArray(
+    const experiments = await collapseKeyIntoArray(
       mainQuery,
       [...fields, 'parent_experiment_id', 'is_subsetted'],
       'key',
       'metadataKeys',
       this.sql,
     );
 
-    return result;
+    const cellLevelMeta = new CellLevelMeta();
+    const experimentIds = experiments.map((experiment) => experiment.id);
+    const cellLevelMetaResults = await cellLevelMeta.getMetadataByExperimentIds(experimentIds);
+
+    cellLevelMetaResults.forEach(
+      (cellLevelMetaResult) => {
+        const experimentIndx = experiments.findIndex(
+          (experiment) => experiment.id === cellLevelMetaResult.experimentId,
+        );
+        experiments[experimentIndx].cellLevelMetadata = cellLevelMetaResult;
+      },
+    );
+
+    return experiments;
   }
 
+
   async getExampleExperiments() {
     const fields = [
       'id',

src/api.v2/model/__mocks__/CellLevelMeta.js

@@ -0,0 +1,10 @@
+const BasicModel = require('./BasicModel')();
+
+const stub = {
+  getMetadataByExperimentIds: () => ([]),
+  ...BasicModel,
+};
+
+const CellLevelMeta = jest.fn().mockImplementation(() => stub);
+
+module.exports = CellLevelMeta;

src/api.v2/model/tableNames.js

@@ -10,6 +10,8 @@ const tableNames = {
   PLOT: 'plot',
   SAMPLE_IN_METADATA_TRACK_MAP: 'sample_in_metadata_track_map',
   SAMPLE_TO_SAMPLE_FILE_MAP: 'sample_to_sample_file_map',
+  CELL_LEVEL: 'cell_metadata_file',
+  CELL_LEVEL_TO_EXPERIMENT_MAP: 'cell_metadata_file_to_experiment',
 };
 
 module.exports = tableNames;

src/api.v2/routes/cellLevel.js

@@ -0,0 +1,16 @@
+const {
+  uploadCellLevelMetadata, updateCellLevelMetadata,
+} = require('../controllers/cellLevelMetaController');
+
+const { expressAuthorizationMiddleware } = require('../middlewares/authMiddlewares');
+
+module.exports = {
+  'cellLevel#uploadCellLevelMetadata': [
+    expressAuthorizationMiddleware,
+    (req, res, next) => uploadCellLevelMetadata(req, res).catch(next),
+  ],
+  'cellLevel#updateCellLevelMetadata': [
+    expressAuthorizationMiddleware,
+    (req, res, next) => updateCellLevelMetadata(req, res).catch(next),
+  ],
+};

src/api.v2/routes/s3Upload.js

@@ -0,0 +1,7 @@
+const { completeMultipartUploads } = require('../controllers/s3Upload');
+
+module.exports = {
+  's3Upload#completeMultipartUpload': [
+    (req, res, next) => completeMultipartUploads(req, res).catch(next),
+  ],
+};

src/api.v2/routes/sampleFile.js

@@ -1,5 +1,5 @@
 const {
-  createFile, patchFile, getS3DownloadUrl, completeMultipart,
+  createFile, patchFile, getS3DownloadUrl,
 } = require('../controllers/sampleFileController');
 
 const { expressAuthorizationMiddleware } = require('../middlewares/authMiddlewares');
@@ -13,9 +13,6 @@ module.exports = {
     expressAuthorizationMiddleware,
     (req, res, next) => patchFile(req, res).catch(next),
   ],
-  'sampleFile#completeMultipart': [
-    (req, res, next) => completeMultipart(req, res).catch(next),
-  ],
   'sampleFile#downloadUrl': [
     expressAuthorizationMiddleware,
     (req, res, next) => getS3DownloadUrl(req, res).catch(next),