Merge pull request #354 from hms-dbmi-cellenics/add-v2-cellsets-endpo…

…ints

[BIOMAGE-1878] - Add V2 cell sets endpoint

codecov.yaml

@@ -11,7 +11,7 @@ coverage:
       default:
         target: auto
         # don't allow new commits to decrease coverage
-        threshold: 1%
+        threshold: 0%
 
     patch:                    # measuring the coverage of new changes
       default:

src/api.v2/controllers/__mocks__/cellSetsController.js

@@ -0,0 +1,7 @@
+const mockGetCellSets = jest.fn();
+const mockPatchCellSets = jest.fn();
+
+module.exports = {
+  getCellSets: mockGetCellSets,
+  patchCellSets: mockPatchCellSets,
+};

src/api.v2/controllers/cellSetsController.js

@@ -0,0 +1,45 @@
+const getLogger = require('../../utils/getLogger');
+
+const getS3Object = require('../helpers/s3/getObject');
+const bucketNames = require('../helpers/s3/bucketNames');
+const formatExperimentId = require('../../utils/v1Compatibility/formatExperimentId');
+
+const { OK } = require('../../utils/responses');
+
+const patchCellSetsObject = require('../helpers/s3/patchCellSetsObject');
+
+const logger = getLogger('[CellSetsController] - ');
+
+const getCellSets = async (req, res) => {
+  let { experimentId } = req.params;
+
+  experimentId = experimentId.replace(/-/g, '');
+
+  logger.log(`Getting cell sets for experiment ${experimentId}`);
+
+  const cellSets = await getS3Object({
+    Bucket: bucketNames.CELL_SETS,
+    Key: formatExperimentId(experimentId),
+  });
+
+  logger.log(`Finished getting cell sets for experiment ${experimentId}`);
+
+  res.send(cellSets);
+};
+
+const patchCellSets = async (req, res) => {
+  const { experimentId } = req.params;
+  const patch = req.body;
+
+  logger.log(`Patching cell sets for ${experimentId}`);
+  await patchCellSetsObject(formatExperimentId(experimentId), patch);
+
+  logger.log(`Finished patching cell sets for experiment ${experimentId}`);
+
+  res.json(OK());
+};
+
+module.exports = {
+  getCellSets,
+  patchCellSets,
+};

src/api.v2/helpers/s3/bucketNames.js

@@ -2,6 +2,7 @@ const config = require('../../../config');
 
 const bucketNames = {
   SAMPLE_FILES: `biomage-originals-${config.clusterEnv}`,
+  CELL_SETS: `cell-sets-${config.clusterEnv}`,
 };
 
 module.exports = bucketNames;

src/api.v2/helpers/s3/getObject.js

@@ -0,0 +1,27 @@
+const getS3Client = require('./getS3Client');
+const NotFoundError = require('../../../utils/responses/NotFoundError');
+
+const getObject = async (params) => {
+  if (!params.Bucket) throw new Error('Bucket is required');
+  if (!params.Key) throw new Error('Key is required');
+
+  const s3 = getS3Client();
+
+  try {
+    const outputObject = await s3.getObject(params).promise();
+    const data = outputObject.Body.toString();
+    return data;
+  } catch (e) {
+    if (e.code === 'NoSuchKey') {
+      throw new NotFoundError(`Couldn't find object with key: ${params.Key}`);
+    }
+
+    if (e.code === 'NoSuchBucket') {
+      throw new NotFoundError(`Couldn't find bucket with key: ${params.Bucket}`);
+    }
+
+    throw e;
+  }
+};
+
+module.exports = getObject;

src/api.v2/helpers/s3/getS3Client.js

@@ -0,0 +1,18 @@
+const AWS = require('../../../utils/requireAWS');
+const config = require('../../../config');
+
+// Wanted to make this a wrapper class that extends S3,
+// but it's not advisable to do so:
+// https://github.com/aws/aws-sdk-js/issues/2006
+const getS3Client = (options) => {
+  const S3Config = {
+    apiVersion: '2006-03-01',
+    signatureVersion: 'v4',
+    region: config.awsRegion,
+    ...options,
+  };
+
+  return new AWS.S3(S3Config);
+};
+
+module.exports = getS3Client;

src/api.v2/helpers/s3/patchCellSetsObject.js

@@ -0,0 +1,39 @@
+const jsonMerger = require('json-merger');
+
+const bucketNames = require('./bucketNames');
+const getObject = require('./getObject');
+const putObject = require('./putObject');
+
+const validateRequest = require('../../../utils/schema-validator');
+
+const patchCellSetsObject = async (experimentId, patch) => {
+  const currentCellSet = await getObject({
+    Bucket: bucketNames.CELL_SETS,
+    Key: experimentId,
+  });
+
+  const { cellSets: prePatchCellSets } = JSON.parse(currentCellSet);
+
+  /**
+   * The $remove operation will replace the element in the array with an
+   * undefined value. We will therefore remove this from the array.
+   *
+   * We use the $remove operation in the worker to update cell clusters,
+   * and we may end up using it in other places in the future.
+   */
+  const patchedCellSetslist = jsonMerger.mergeObjects(
+    [prePatchCellSets, patch],
+  );
+
+  const patchedCellSets = { cellSets: patchedCellSetslist };
+
+  await validateRequest(patchedCellSets, 'cell-sets-bodies/CellSets.v2.yaml');
+
+  await putObject({
+    Bucket: bucketNames.CELL_SETS,
+    Key: experimentId,
+    Body: JSON.stringify(patchedCellSets),
+  });
+};
+
+module.exports = patchCellSetsObject;

src/api.v2/helpers/s3/putObject.js

@@ -0,0 +1,22 @@
+const NotFoundError = require('../../../utils/responses/NotFoundError');
+const getS3Client = require('./getS3Client');
+
+const pubObject = async (params) => {
+  if (!params.Bucket) throw new Error('Bucket is required');
+  if (!params.Key) throw new Error('Key is required');
+  if (!params.Body) throw new Error('Body is required');
+
+  const s3 = getS3Client();
+
+  try {
+    await s3.putObject(params).promise();
+  } catch (e) {
+    if (e.code === 'NoSuchBucket') {
+      throw new NotFoundError(`Couldn't find bucket with key: ${params.Bucket}`);
+    }
+
+    throw e;
+  }
+};
+
+module.exports = pubObject;

src/api.v2/routes/cellSets.js

@@ -0,0 +1,17 @@
+const {
+  getCellSets,
+  patchCellSets,
+} = require('../controllers/cellSetsController');
+
+const { expressAuthorizationMiddleware } = require('../middlewares/authMiddlewares');
+
+module.exports = {
+  'cellSets#getCellSets': [
+    expressAuthorizationMiddleware,
+    (req, res, next) => getCellSets(req, res).catch(next),
+  ],
+  'cellSets#patchCellSets': [
+    expressAuthorizationMiddleware,
+    (req, res, next) => patchCellSets(req, res).catch(next),
+  ],
+};

src/specs/api.v2.yaml

@@ -58,6 +58,84 @@ paths:
                   - env
                   - clusterEnv
       description: Returns a status on the health of the API.
+
+  '/experiments/{experimentId}/cellSets':
+    parameters:
+      - name: experimentId
+        in: path
+        description: ID of experiment to find cell sets of.
+        required: true
+        schema:
+          type: string
+    get:
+      tags:
+        - experiments
+      summary: Get cell sets for experiment
+      description: Returns a hirearchical view of cell sets in the experiment.
+      operationId: getExperimentCellSets
+      x-eov-operation-id: cellSets#getCellSets
+      x-eov-operation-handler: routes/cellSets
+      responses:
+        '200':
+          description: 'Request successful, hierarchy returned below.'
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/CellSets'
+        '401':
+          description: The request lacks authentication credentials.
+          content:
+            application/json:
+              schema:
+                $ref: ./models/HTTPError.v1.yaml
+        '403':
+          description: The authenticated user is not authorized to view this resource.
+          content:
+            application/json:
+              schema:
+                $ref: ./models/HTTPError.v1.yaml
+        '404':
+          description: Experiment not found.
+          content:
+            application/json:
+              schema:
+                $ref: ./models/HTTPError.v1.yaml
+    patch:
+      summary: ''
+      operationId: patchExperimentCellSets
+      x-eov-operation-id: cellSets#patchCellSets
+      x-eov-operation-handler: routes/cellSets
+      responses:
+        '200':
+          description: Update to object in response successful.
+          content:
+            application/json:
+              schema:
+                type: object
+                properties: {}
+        '401':
+          description: The request lacks authentication credentials.
+          content:
+            application/json:
+              schema:
+                $ref: ./models/HTTPError.v1.yaml
+        '403':
+          description: The authenticated user is not authorized to view this resource.
+          content:
+            application/json:
+              schema:
+                $ref: ./models/HTTPError.v1.yaml
+      description: Performs a partial update on the experiment's cell sets.
+      requestBody:
+        content:
+          application/boschni-json-merger+json:
+            schema:
+              oneOf:
+                - type: array
+                  items: {}
+                - type: object
+        description: The patch in the format declared by boschni/json-merger. Note the explicit naming of the content subtype (boschni-json-merger+json) which must be used to explicitly acknowledge the format.
+
   '/experiments/{experimentId}/processingConfig':
     get:
       summary: Get processing configuration for an experiment
@@ -828,7 +906,7 @@ paths:
             application/json:
               schema:
                 $ref: '#/components/schemas/HTTPError'
-                
+
   '/access/{experimentId}':
     get:
       summary: Get the users with access to an experiment
@@ -904,6 +982,8 @@ components:
       $ref: './models/samples-bodies/CreateSampleFile.v2.yaml'
     PatchSampleFile:
       $ref: './models/samples-bodies/PatchSampleFile.v2.yaml'
+    CellSets:
+      $ref: ./models/cell-sets-bodies/CellSets.v2.yaml
     HTTPSuccess:
       $ref: './models/HTTPSuccess.v1.yaml'
     HTTPError:

src/specs/models/cell-sets-bodies/CellSet.v2.yaml

@@ -0,0 +1,18 @@
+title: Cell Set
+description: An object representing a cell set (e.g. Cluster 1)
+type: object
+properties:
+  key:
+    type: string
+  name:
+    type: string
+  rootNode:
+    type: boolean
+  cellIds:
+    type: array
+    items:
+      type: integer
+required:
+  - key
+  - name
+  - cellIds

src/specs/models/cell-sets-bodies/CellSetClass.v2.yaml

@@ -0,0 +1,18 @@
+title: Cell Set
+description: An object representing a cell set (e.g. Cluster 1)
+type: object
+properties:
+  key:
+    type: string
+  name:
+    type: string
+  rootNode:
+    type: boolean
+  children:
+    type: array
+    items:
+      $ref: ./CellSet.v2.yaml
+required:
+  - key
+  - name
+  - children

src/specs/models/cell-sets-bodies/CellSets.v2.yaml

@@ -0,0 +1,10 @@
+title: Cell Sets object
+description: Schema for CellSets object
+type: object
+properties:
+  cellSets:
+    type: array
+    items:
+      $ref: ./CellSetClass.v2.yaml
+required:
+  - cellSets

src/utils/schema-validator.js

@@ -6,6 +6,8 @@ const Validator = require('swagger-model-validator');
 const yaml = require('js-yaml');
 const _ = require('lodash');
 
+const BadRequestError = require('./responses/BadRequestError');
+
 const getLogger = require('./getLogger');
 
 const logger = getLogger();
@@ -42,7 +44,7 @@ const validateRequest = async (request, schemaPath) => {
 
   if (!validation.valid) {
     logger.log(`Validation error for: ${request}`);
-    throw new Error(validation.errors[0]);
+    throw new BadRequestError(validation.errors[0]);
   }
 };
 

src/utils/v1Compatibility/formatExperimentId.js

@@ -0,0 +1,9 @@
+// Existing experimentId in S3 (v1) are MD5 hashes not UUIDs.
+// They have the same number of alhpanum characters as UUIDs but no dashes
+// To maintain compatibility with v1, we remove the dashes from UUIDs.
+// This function should be removed once we have migrated to v2.
+// TODO: migrate existing cellsets to the dashes uuidv4 format so this function is not necessary
+
+const formatExperimentId = (experimentId) => experimentId.replace(/-/g, '');
+
+module.exports = formatExperimentId;