changes 5

.github/workflows/ci.yaml

@@ -335,7 +335,6 @@ jobs:
           REF_ID: ${{ needs.build-docker.outputs.ref-id }}
           IMAGE_TAG: ${{ needs.build-docker.outputs.image-tag }}
           REGISTRY: ${{ steps.login-ecr.outputs.registry }}
-
       - name: Push production/develop template to releases
         if:
           (matrix.environment == 'production' && github.event_name == 'release' && github.event.action == 'released') || (matrix.environment == 'develop' && github.event_name == 'push')
@@ -370,7 +369,17 @@ jobs:
           destination_folder: 'staging-candidates/${{ steps.fill-metadata.outputs.deployment-name }}'
           user_email: '[email protected]'
           user_name: 'Biomage CI/CD'
-
+      - id: disable-admin-enforcement
+        if:
+          (matrix.environment == 'production' && github.event_name == 'release' && github.event.action == 'released') || (matrix.environment == 'develop' && github.event_name == 'push')
+        name: Temporarily disable admin enforcement
+        uses: benjefferies/branch-protection-bot@master
+        with:
+          access_token: ${{ secrets.API_TOKEN_GITHUB }}
+          owner: ${{ github.repository_owner }}
+          repo: iac
+          enforce_admins: false
+          retries: 8
       - name: Push migrations into iac
         if:
           (matrix.environment == 'production' && github.event_name == 'release' && github.event.action == 'released') || (matrix.environment == 'develop' && github.event_name == 'push')

.github/workflows/pr_validate.yaml

@@ -28,7 +28,7 @@ jobs:
       - id: check-issue-url
         name: Check issue URL
         run: |-
-          REGEX="#### URL to issue\s?\r\n(https:\/\/biomage\.atlassian\.net\/browse\/BIOMAGE-[0-9]+|N\/A)"
+          REGEX="#### URL to issue\s?\r\n(https:\/\/github\.com\/hms-dbmi-cellenics\/issues\/issues\/[0-9]+|N\/A)"
 
           echo "REGEX to test against:"
           echo $REGEX
@@ -41,7 +41,7 @@ jobs:
       - id: extract-staging
         name: Check staging URL
         run: |-
-          REGEX="#### Link to staging deployment URL \(or set N\/A\)\s?\r\n(https:\/\/ui-(.+)\.scp-staging\.biomage\.net|N\/A)"
+          REGEX="#### Link to staging deployment URL \(or set N\/A\)\s?\r\n(https:\/\/ui-(.+)\.staging\.single-cell-platform\.net|N\/A)"
 
           echo "REGEX to test against:"
           echo $REGEX

Makefile

@@ -7,7 +7,7 @@
 # Targets
 #--------------------------------------------------
 install: ## Installs node dependencies
-	@npm install
+	@rm -rf node_modules && npm ci
 check: ## Checks code for linting/construct errors
 	@echo "==> Checking if files are well formatted..."
 	@npm run lint

src/api.v2/constants.js

@@ -4,6 +4,7 @@ const GEM2S_PROCESS_NAME = 'gem2s';
 const SEURAT_PROCESS_NAME = 'seurat';
 const OLD_QC_NAME_TO_BE_REMOVED = 'pipeline';
 const SUBSET_PROCESS_NAME = 'subset';
+const COPY_PROCESS_NAME = 'copy';
 
 // Pipeline task names
 const ASSIGN_POD_TO_PIPELINE = 'assignPodToPipeline';
@@ -52,6 +53,7 @@ module.exports = {
   SEURAT_PROCESS_NAME,
   OLD_QC_NAME_TO_BE_REMOVED,
   SUBSET_PROCESS_NAME,
+  COPY_PROCESS_NAME,
   PIPELINE_ERROR,
   HANDLE_ERROR_STEP,
   END_OF_PIPELINE,

src/api.v2/controllers/__mocks__/accessController.js

@@ -2,10 +2,12 @@ const mockGetUserAccess = jest.fn();
 const mockInviteUser = jest.fn();
 const mockRevokeAccess = jest.fn();
 const mockPostRegistration = jest.fn();
+const mockIsUserAuthorized = jest.fn();
 
 module.exports = {
   getUserAccess: mockGetUserAccess,
   inviteUser: mockInviteUser,
   revokeAccess: mockRevokeAccess,
   postRegistration: mockPostRegistration,
+  isUserAuthorized: mockIsUserAuthorized,
 };

src/api.v2/controllers/accessController.js

@@ -5,6 +5,7 @@ const postRegistrationHandler = require('../helpers/access/postRegistrationHandl
 
 const OK = require('../../utils/responses/OK');
 const getLogger = require('../../utils/getLogger');
+const UserAccess = require('../model/UserAccess');
 
 const logger = getLogger('[AccessController] - ');
 
@@ -54,9 +55,22 @@ const postRegistration = async (req, res) => {
   res.json(OK());
 };
 
+const isUserAuthorized = async (req, res) => {
+  const { params: { experimentId }, query: { url, method }, user: { sub: userId } } = req;
+
+  logger.log(`Checking access permissions for ${userId}`);
+
+  const result = await (new UserAccess().canAccessExperiment(userId, experimentId, url, method));
+
+  logger.log(`Finished, returning access permissions for ${userId}`);
+
+  res.json(result);
+};
+
 module.exports = {
   getUserAccess,
   inviteUser,
   revokeAccess,
   postRegistration,
+  isUserAuthorized,
 };

src/api.v2/controllers/experimentController.js

@@ -13,14 +13,33 @@ const invalidatePlotsForEvent = require('../../utils/plotConfigInvalidation/inva
 const events = require('../../utils/plotConfigInvalidation/events');
 const getAdminSub = require('../../utils/getAdminSub');
 const config = require('../../config');
+const ExperimentExecution = require('../model/ExperimentExecution');
+const Plot = require('../model/Plot');
+const { createCopyPipeline } = require('../helpers/pipeline/pipelineConstruct');
+const { OLD_QC_NAME_TO_BE_REMOVED, NOT_CREATED } = require('../constants');
+const { RUNNING } = require('../constants');
+const { GEM2S_PROCESS_NAME } = require('../constants');
+const LockedError = require('../../utils/responses/LockedError');
 
 const logger = getLogger('[ExperimentController] - ');
 
+const translateProcessingConfig = (processingConfig, sampleIdsMap) => (
+  _.transform(processingConfig, (acc, value, key) => {
+    // If the key is a sample id, then replace it with the new id
+    const newKey = sampleIdsMap[key] || key;
+
+    // Keep going and translate the rest of the object
+    acc[newKey] = _.isObject(value)
+      ? translateProcessingConfig(value, sampleIdsMap)
+      : value;
+  })
+);
+
 const getDefaultCPUMem = (env) => {
   switch (env) {
     case 'staging':
       return { podCPUs: 1, podMemory: 14000 };
-      // Stop using Batch by default in 'production':
+    // Stop using Batch by default in 'production':
     //   return { podCPUs: 2, podMemory: 28000 };
     default:
       return { podCPUs: null, podMemory: null };
@@ -166,20 +185,11 @@ const downloadData = async (req, res) => {
   res.json(downloadLink);
 };
 
-
 const cloneExperiment = async (req, res) => {
-  const getAllSampleIds = async (experimentId) => {
-    const { samplesOrder } = await new Experiment().findById(experimentId).first();
-    return samplesOrder;
-  };
   const userId = req.user.sub;
   const {
     params: { experimentId: fromExperimentId },
-    body: {
-      samplesToCloneIds = await getAllSampleIds(fromExperimentId),
-      name = null,
-      toUserId = userId,
-    },
+    body: { toUserId = userId, name },
   } = req;
 
   const adminSub = await getAdminSub();
@@ -188,6 +198,15 @@ const cloneExperiment = async (req, res) => {
     throw new UnauthorizedError(`User ${userId} cannot clone experiments for other users.`);
   }
 
+  const {
+    [OLD_QC_NAME_TO_BE_REMOVED]: { status: qcStatus },
+    [GEM2S_PROCESS_NAME]: { status: gem2sStatus },
+  } = await getExperimentBackendStatus(fromExperimentId);
+
+  if (qcStatus === RUNNING || gem2sStatus === RUNNING) {
+    throw new LockedError('Experiment is currently running a pipeline and can\'t be copied');
+  }
+
   logger.log(`Creating experiment to clone ${fromExperimentId} to`);
 
   let toExperimentId;
@@ -197,18 +216,48 @@ const cloneExperiment = async (req, res) => {
     await new UserAccess(trx).createNewExperimentPermissions(toUserId, toExperimentId);
   });
 
-  logger.log(`Cloning experiment samples from experiment ${fromExperimentId} into ${toExperimentId}`);
+  const { samplesOrder: samplesToCloneIds, processingConfig } = await new Experiment()
+    .findById(fromExperimentId)
+    .first();
 
-  const cloneSamplesOrder = await new Sample().copyTo(
-    fromExperimentId, toExperimentId, samplesToCloneIds,
-  );
+  const cloneSamplesOrder = await new Sample()
+    .copyTo(fromExperimentId, toExperimentId, samplesToCloneIds);
+
+  // Group together the original and copy sample ids for cleaner handling
+  const sampleIdsMap = _.zipObject(samplesToCloneIds, cloneSamplesOrder);
+
+  const translatedProcessingConfig = translateProcessingConfig(processingConfig, sampleIdsMap);
 
   await new Experiment().updateById(
     toExperimentId,
-    { samples_order: JSON.stringify(cloneSamplesOrder) },
+    {
+      samples_order: JSON.stringify(cloneSamplesOrder),
+      processing_config: JSON.stringify(translatedProcessingConfig),
+    },
+  );
+
+  // If the experiment didn't run yet, there's nothing else to update
+  if (gem2sStatus === NOT_CREATED) {
+    logger.log(`Finished cloning ${fromExperimentId}, no pipeline to run because experiment never ran`);
+
+    res.json(toExperimentId);
+    return;
+  }
+
+  await new ExperimentExecution().copyTo(fromExperimentId, toExperimentId, sampleIdsMap);
+  await new Plot().copyTo(fromExperimentId, toExperimentId, sampleIdsMap);
+
+  const {
+    stateMachineArn,
+    executionArn,
+  } = await createCopyPipeline(fromExperimentId, toExperimentId, sampleIdsMap);
+
+  await new ExperimentExecution().upsert(
+    { experiment_id: toExperimentId, pipeline_type: 'gem2s' },
+    { state_machine_arn: stateMachineArn, execution_arn: executionArn },
   );
 
-  logger.log(`Finished cloning experiment ${fromExperimentId}, new experiment's id is ${toExperimentId}`);
+  logger.log(`Began pipeline for cloning experiment ${fromExperimentId}, new experiment's id is ${toExperimentId}`);
 
   res.json(toExperimentId);
 };

src/api.v2/controllers/gem2sController.js

@@ -17,7 +17,6 @@ const runGem2s = async (req, res) => {
   const { parentExperimentId = null } = await new ExperimentParent()
     .find({ experiment_id: experimentId })
     .first();
-
   if (parentExperimentId) {
     throw new MethodNotAllowedError(`Experiment ${experimentId} can't run gem2s`);
   }

src/api.v2/helpers/access/createUserInvite.js

@@ -19,7 +19,7 @@ const createUserInvite = async (experimentId, invitedUserEmail, role, inviterUse
     userAttributes = await getAwsUserAttributesByEmail(invitedUserEmail);
 
     const invitedUserId = userAttributes.find((attr) => attr.Name === 'sub').Value;
-    new UserAccess().grantAccess(invitedUserId, experimentId, role);
+    await new UserAccess().grantAccess(invitedUserId, experimentId, role);
     emailBody = buildUserInvitedEmailBody(invitedUserEmail, experimentId, inviterUser);
   } catch (e) {
     if (e.code !== 'UserNotFoundException') {
@@ -28,11 +28,17 @@ const createUserInvite = async (experimentId, invitedUserEmail, role, inviterUse
 
     logger.log('Invited user does not have an account yet. Sending invitation email.');
 
-    new UserAccess().addToInviteAccess(invitedUserEmail, experimentId, role);
+    await new UserAccess().addToInviteAccess(invitedUserEmail, experimentId, role);
     emailBody = buildUserInvitedNotRegisteredEmailBody(invitedUserEmail, inviterUser);
   }
 
-  await sendEmail(emailBody);
+  try {
+    await sendEmail(emailBody);
+  } catch (e) {
+    logger.error(e);
+    // This message is picked up in the ui and transformed to a nice end user message
+    throw new Error('NotificationFailure');
+  }
 
   return OK();
 };

src/api.v2/helpers/access/postRegistrationHandler.js

@@ -1,32 +1,18 @@
-const parseSNSMessage = require('../../../utils/parseSNSMessage');
 const getLogger = require('../../../utils/getLogger');
+const { OK } = require('../../../utils/responses');
 
 const UserAccess = require('../../model/UserAccess');
-const snsTopics = require('../../../config/snsTopics');
 
 const logger = getLogger('[PostRegistrationHandler] - ');
 
 const postRegistrationHandler = async (req) => {
-  let data;
-  let messageType;
-
-  try {
-    const { parsedMessage, msg } = await parseSNSMessage(req, snsTopics.POST_REGISTRATION);
-    data = parsedMessage;
-    messageType = msg.Type;
-  } catch (e) {
-    logger.error('Parsing initial SNS message failed:', e);
-    return;
-  }
-
-  // If this is a subscription confirmation, we can just return.
-  if (messageType !== 'Notification') return;
-
-  const { userEmail, userId } = data;
+  const { userEmail, userId } = req.body;
 
   new UserAccess().registerNewUserAccess(userEmail, userId);
 
   logger.log(`Post registration handled for user ${userId}`);
+
+  return OK();
 };
 
 module.exports = postRegistrationHandler;

src/api.v2/helpers/pipeline/gem2s.js

@@ -17,6 +17,8 @@ const getLogger = require('../../../utils/getLogger');
 
 const { qcStepsWithFilterSettings } = require('./pipelineConstruct/qcHelpers');
 const { getPipelineParams, formatSamples } = require('./shouldPipelineRerun');
+const invalidatePlotsForEvent = require('../../../utils/plotConfigInvalidation/invalidatePlotsForEvent');
+const events = require('../../../utils/plotConfigInvalidation/events');
 
 const logger = getLogger('[Gem2sService] - ');
 
@@ -122,8 +124,13 @@ const setupSubsetSamples = async (payload) => {
   // Add samples that were created
 };
 
+const invalidatePlotsForExperiment = async (payload, io) => {
+  await invalidatePlotsForEvent(payload.experimentId, events.CELL_SETS_MODIFIED, io.sockets);
+};
+
 hookRunner.register('subsetSeurat', [setupSubsetSamples]);
 hookRunner.register('uploadToAWS', [continueToQC]);
+hookRunner.register('copyS3Objects', [invalidatePlotsForExperiment]);
 
 hookRunner.registerAll([sendNotification]);
 
@@ -232,7 +239,7 @@ const handleGem2sResponse = async (io, message) => {
   // Fail hard if there was an error.
   await validateRequest(message, 'GEM2SResponse.v2.yaml');
 
-  await hookRunner.run(message);
+  await hookRunner.run(message, io);
 
   const { experimentId } = message;
 

src/api.v2/helpers/pipeline/handleQCResponse.js

@@ -140,7 +140,7 @@ const handleQCResponse = async (io, message) => {
 
   await validateRequest(message, 'PipelineResponse.v2.yaml');
 
-  await hookRunner.run(message);
+  await hookRunner.run(message, io);
 
   const { experimentId, input: { sampleUuid, taskName } } = message;
   const { error = false } = message.response || {};

src/api.v2/helpers/pipeline/hooks/HookRunner.js

@@ -30,7 +30,7 @@ class HookRunner {
   }
 
   // run requires taskName to be present as a key in the payload
-  async run(payload) {
+  async run(payload, io) {
     // run all hooks inside a try catch because they are side-effects and as such, they should
     // not break the main program execution
     try {
@@ -45,13 +45,13 @@ class HookRunner {
       for (let i = 0; this.hooks[taskName] !== undefined && i < this.hooks[taskName].length; i += 1) {
         // calling the hooks sequentially since they may depend on each other
         // eslint-disable-next-line no-await-in-loop
-        this.results[taskName].push(await this.hooks[taskName][i](payload));
+        this.results[taskName].push(await this.hooks[taskName][i](payload, io));
       }
 
       // Runs hooks that apply to all tasks (assigning the results to current task)
       for (let i = 0; this.hooks[ALL] !== undefined && i < this.hooks[ALL].length; i += 1) {
         // eslint-disable-next-line no-await-in-loop
-        this.results[taskName].push(await this.hooks[ALL][i](payload));
+        this.results[taskName].push(await this.hooks[ALL][i](payload, io));
       }
 
       logger.log(`Completed ${this.results[taskName].length} hooks for pipeline task ${taskName}`);