Merge pull request #431 from hms-dbmi-cellenics/biomage-changes-3

Bump workerVersion to invalidate work results

src/api.v2/helpers/pipeline/getPipelineStatus.js

@@ -37,7 +37,7 @@ const gem2sPipelineSteps = [
 const seuratPipelineSteps = [
   'DownloadSeurat',
   'ProcessSeurat',
-  'UploadSeuratToAWS'
+  'UploadSeuratToAWS',
 ];
 
 // pipelineStepNames are the names of pipeline steps for which we
@@ -292,7 +292,7 @@ const getPipelineStatus = async (experimentId, processName) => {
   let response = null;
 
   const { executionArn = null, stateMachineArn = null, lastStatusResponse } = pipelineExecution;
-    const shouldRerun = await shouldPipelineRerun(experimentId, processName);
+  const shouldRerun = await shouldPipelineRerun(experimentId, processName);
 
   try {
     execution = await stepFunctions.describeExecution({

src/api.v2/helpers/pipeline/pipelineConstruct/index.js

@@ -12,8 +12,12 @@ const ExperimentExecution = require('../../../model/ExperimentExecution');
 const getLogger = require('../../../../utils/getLogger');
 
 const {
-  getGem2sPipelineSkeleton, getQcPipelineSkeleton, getSubsetPipelineSkeleton, getSeuratPipelineSkeleton,
+  getGem2sPipelineSkeleton,
+  getQcPipelineSkeleton,
+  getSubsetPipelineSkeleton,
+  getSeuratPipelineSkeleton,
 } = require('./skeletons');
+
 const { getQcStepsToRun, qcStepsWithFilterSettings } = require('./qcHelpers');
 const needsBatchJob = require('../batch/needsBatchJob');
 

src/api.v2/helpers/pipeline/shouldPipelineRerun.js

@@ -90,7 +90,9 @@ const getPipelineParams = async (experimentId, rawSamples = undefined) => {
 };
 
 const shouldPipelineRerun = async (experimentId, pipelineType) => {
-  const execution = await new ExperimentExecution().findOne({ experiment_id: experimentId, pipeline_type: pipelineType });
+  const execution = await new ExperimentExecution()
+    .findOne({ experiment_id: experimentId, pipeline_type: pipelineType });
+
   if (execution === undefined) return true;
   const currentParams = await getPipelineParams(experimentId);
 

src/config/default-config.js

@@ -76,7 +76,7 @@ const config = {
   // Used for Batch reporting
   datadogApiKey: process.env.DD_API_KEY || '',
   datadogAppKey: process.env.DD_APP_KEY || '',
-  workerVersion: 3, // needs to match workerVersion in UI
+  workerVersion: 4, // needs to match workerVersion in UI
 };
 
 // We are in permanent develop staging environment

src/config/test-config.js

@@ -24,5 +24,5 @@ module.exports = {
   pipelineIgnoreSSLCertificate: false,
   datadogApiKey: 'test-datadog-api-key', // pragma: allowlist secret
   datadogAppKey: 'test-datadog-app-key',
-  workerVersion: 3, // needs to match workerVersion in UI
+  workerVersion: 4, // needs to match workerVersion in UI
 };

src/sql/migrations/20230117130009_add_last_pipeline_params.js

@@ -1,7 +1,12 @@
 exports.up = async (knex) => {
-  await knex.schema.alterTable('experiment_execution', (table) => {
-    table.renameColumn('last_gem2s_params', 'last_pipeline_params');
-  });
+  // for HMS deployment last_gem2s_params was always called last_pipeline_params
+  const hasLastGem2sParams = await knex.schema.hasColumn('experiment_execution', 'last_gem2s_params');
+
+  if (hasLastGem2sParams) {
+    await knex.schema.alterTable('experiment_execution', (table) => {
+      table.renameColumn('last_gem2s_params', 'last_pipeline_params');
+    });
+  }
 };
 
 exports.down = async (knex) => {

tests/api.v2/helpers/worker/__snapshots__/submitEmbeddingWork.test.js.snap

@@ -17,7 +17,7 @@ Array [
       "extraDependencies": Array [],
       "extras": undefined,
       "qcPipelineStartDate": "2016-03-22T04:00:00.000Z",
-      "workerVersion": 3,
+      "workerVersion": 4,
     },
   ],
 ]

tests/api.v2/helpers/worker/__snapshots__/submitMarkerHeatmapWork.test.js.snap

@@ -29,7 +29,7 @@ Array [
       ],
       "extras": undefined,
       "qcPipelineStartDate": "2016-03-22T04:00:00.000Z",
-      "workerVersion": 3,
+      "workerVersion": 4,
     },
   ],
 ]

tests/api.v2/helpers/worker/submitEmbeddingWork.test.js

@@ -57,7 +57,7 @@ describe('submitWorkEmbedding', () => {
     expect(createObjectHash.mock.calls).toMatchSnapshot();
     // this ETag should match exactly the one in
     // loadEmbedding.defaultParams.test.js
-    expect(ETag).toEqual('fb7a8a67c7806fa90c69b9cdb16dd478'); // pragma: allowlist secret
+    expect(ETag).toEqual('163301323ab0ee82902d1574a5dcd061'); // pragma: allowlist secret
     expect(validateAndSubmitWork).toBeCalledTimes(1);
   });
 });

tests/api.v2/helpers/worker/submitMarkerHeatmapWork.test.js

@@ -57,7 +57,7 @@ describe('submitWorkEmbedding', () => {
     expect(createObjectHash.mock.calls).toMatchSnapshot();
     // this ETag should match exactly the one in
     // loadMarkerGenes.defaultParams.test.js
-    expect(ETag).toEqual('a973357e8c6273f0f154a179b12c6d25'); // pragma: allowlist secret
+    expect(ETag).toEqual('7f2b831012fe0d6bebf8fcca49ccd057'); // pragma: allowlist secret
 
     expect(validateAndSubmitWork).toBeCalledTimes(1);
   });

tests/config/__snapshots__/default-config-production.test.js.snap

@@ -33,7 +33,7 @@ Object {
   "rdsSandboxId": "default",
   "sandboxId": "default",
   "workerNamespace": "worker-default",
-  "workerVersion": 3,
+  "workerVersion": 4,
 }
 `;
 
@@ -69,6 +69,6 @@ Object {
   "rdsSandboxId": "default",
   "sandboxId": "default",
   "workerNamespace": "worker-default",
-  "workerVersion": 3,
+  "workerVersion": 4,
 }
 `;

tests/config/__snapshots__/default-config-staging-sandboxid.test.js.snap

@@ -33,6 +33,6 @@ Object {
   "rdsSandboxId": "default",
   "sandboxId": "mockedSandboxId",
   "workerNamespace": "worker-mockedSandboxId",
-  "workerVersion": 3,
+  "workerVersion": 4,
 }
 `;

tests/config/__snapshots__/default-config-staging.test.js.snap

@@ -33,6 +33,6 @@ Object {
   "rdsSandboxId": "default",
   "sandboxId": "default",
   "workerNamespace": "worker-default",
-  "workerVersion": 3,
+  "workerVersion": 4,
 }
 `;