From b260508a4fabcc1b07d3a20b8ebd88d9f1289ea0 Mon Sep 17 00:00:00 2001 From: Alex Pickering Date: Tue, 28 Mar 2023 15:18:46 -0700 Subject: [PATCH 1/3] check for last_gem2_params Signed-off-by: Alex Pickering --- .../20230117130009_add_last_pipeline_params.js | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/sql/migrations/20230117130009_add_last_pipeline_params.js b/src/sql/migrations/20230117130009_add_last_pipeline_params.js index f5fb7538d..083e1b366 100644 --- a/src/sql/migrations/20230117130009_add_last_pipeline_params.js +++ b/src/sql/migrations/20230117130009_add_last_pipeline_params.js @@ -1,7 +1,11 @@ exports.up = async (knex) => { - await knex.schema.alterTable('experiment_execution', (table) => { - table.renameColumn('last_gem2s_params', 'last_pipeline_params'); - }); + const hasLastGem2sParams = await knex.schema.hasColumn('experiment_execution', 'last_gem2s_params'); + + if (hasLastGem2sParams) { + await knex.schema.alterTable('experiment_execution', (table) => { + table.renameColumn('last_gem2s_params', 'last_pipeline_params'); + }); + } }; exports.down = async (knex) => { From fabad5bba9c80c445bd3c011a6548b629063f470 Mon Sep 17 00:00:00 2001 From: Alex Pickering Date: Wed, 29 Mar 2023 11:56:02 -0700 Subject: [PATCH 2/3] bump worker version Signed-off-by: Alex Pickering --- src/config/default-config.js | 2 +- src/config/test-config.js | 2 +- ...20230117130009_add_last_pipeline_params.js | 24 ++++++++++++------- .../submitEmbeddingWork.test.js.snap | 2 +- .../submitMarkerHeatmapWork.test.js.snap | 2 +- .../default-config-production.test.js.snap | 4 ++-- ...ault-config-staging-sandboxid.test.js.snap | 2 +- .../default-config-staging.test.js.snap | 2 +- 8 files changed, 24 insertions(+), 16 deletions(-) diff --git a/src/config/default-config.js b/src/config/default-config.js index c30e2d80e..f528ee0b4 100644 --- a/src/config/default-config.js +++ b/src/config/default-config.js @@ -76,7 +76,7 @@ const config = { // Used for Batch reporting datadogApiKey: process.env.DD_API_KEY || '', datadogAppKey: process.env.DD_APP_KEY || '', - workerVersion: 3, // needs to match workerVersion in UI + workerVersion: 4, // needs to match workerVersion in UI }; // We are in permanent develop staging environment diff --git a/src/config/test-config.js b/src/config/test-config.js index b6d68de09..02f28bebf 100644 --- a/src/config/test-config.js +++ b/src/config/test-config.js @@ -24,5 +24,5 @@ module.exports = { pipelineIgnoreSSLCertificate: false, datadogApiKey: 'test-datadog-api-key', // pragma: allowlist secret datadogAppKey: 'test-datadog-app-key', - workerVersion: 3, // needs to match workerVersion in UI + workerVersion: 4, // needs to match workerVersion in UI }; diff --git a/src/sql/migrations/20230117130009_add_last_pipeline_params.js b/src/sql/migrations/20230117130009_add_last_pipeline_params.js index 083e1b366..30180cf8a 100644 --- a/src/sql/migrations/20230117130009_add_last_pipeline_params.js +++ b/src/sql/migrations/20230117130009_add_last_pipeline_params.js @@ -1,15 +1,23 @@ -exports.up = async (knex) => { - const hasLastGem2sParams = await knex.schema.hasColumn('experiment_execution', 'last_gem2s_params'); +exports.up = (knex) => { + const hasLastGem2sParams = knex.schema.hasColumn('experiment_execution', 'last_gem2s_params'); + + knex.schema.hasColumn('experiment_execution', 'last_gem2s_params') + .then((exists) => { + console.log('exists'); + console.log(exists); + }); + + console.log('hasLastGem2sParams'); + console.log(hasLastGem2sParams); if (hasLastGem2sParams) { - await knex.schema.alterTable('experiment_execution', (table) => { + console.log('here!!!'); + return knex.schema.alterTable('experiment_execution', (table) => { table.renameColumn('last_gem2s_params', 'last_pipeline_params'); }); } }; -exports.down = async (knex) => { - await knex.schema.alterTable('experiment_execution', (table) => { - table.renameColumn('last_pipeline_params', 'last_gem2s_params'); - }); -}; +exports.down = (knex) => knex.schema.alterTable('experiment_execution', (table) => { + table.renameColumn('last_pipeline_params', 'last_gem2s_params'); +}); diff --git a/tests/api.v2/helpers/worker/__snapshots__/submitEmbeddingWork.test.js.snap b/tests/api.v2/helpers/worker/__snapshots__/submitEmbeddingWork.test.js.snap index c87840138..f92ed2200 100644 --- a/tests/api.v2/helpers/worker/__snapshots__/submitEmbeddingWork.test.js.snap +++ b/tests/api.v2/helpers/worker/__snapshots__/submitEmbeddingWork.test.js.snap @@ -17,7 +17,7 @@ Array [ "extraDependencies": Array [], "extras": undefined, "qcPipelineStartDate": "2016-03-22T04:00:00.000Z", - "workerVersion": 3, + "workerVersion": 4, }, ], ] diff --git a/tests/api.v2/helpers/worker/__snapshots__/submitMarkerHeatmapWork.test.js.snap b/tests/api.v2/helpers/worker/__snapshots__/submitMarkerHeatmapWork.test.js.snap index f5c7bd6e9..4e6979416 100644 --- a/tests/api.v2/helpers/worker/__snapshots__/submitMarkerHeatmapWork.test.js.snap +++ b/tests/api.v2/helpers/worker/__snapshots__/submitMarkerHeatmapWork.test.js.snap @@ -29,7 +29,7 @@ Array [ ], "extras": undefined, "qcPipelineStartDate": "2016-03-22T04:00:00.000Z", - "workerVersion": 3, + "workerVersion": 4, }, ], ] diff --git a/tests/config/__snapshots__/default-config-production.test.js.snap b/tests/config/__snapshots__/default-config-production.test.js.snap index a3a0efc60..c7a4514fb 100644 --- a/tests/config/__snapshots__/default-config-production.test.js.snap +++ b/tests/config/__snapshots__/default-config-production.test.js.snap @@ -33,7 +33,7 @@ Object { "rdsSandboxId": "default", "sandboxId": "default", "workerNamespace": "worker-default", - "workerVersion": 3, + "workerVersion": 4, } `; @@ -69,6 +69,6 @@ Object { "rdsSandboxId": "default", "sandboxId": "default", "workerNamespace": "worker-default", - "workerVersion": 3, + "workerVersion": 4, } `; diff --git a/tests/config/__snapshots__/default-config-staging-sandboxid.test.js.snap b/tests/config/__snapshots__/default-config-staging-sandboxid.test.js.snap index beab39490..99a810be4 100644 --- a/tests/config/__snapshots__/default-config-staging-sandboxid.test.js.snap +++ b/tests/config/__snapshots__/default-config-staging-sandboxid.test.js.snap @@ -33,6 +33,6 @@ Object { "rdsSandboxId": "default", "sandboxId": "mockedSandboxId", "workerNamespace": "worker-mockedSandboxId", - "workerVersion": 3, + "workerVersion": 4, } `; diff --git a/tests/config/__snapshots__/default-config-staging.test.js.snap b/tests/config/__snapshots__/default-config-staging.test.js.snap index 51101ce4a..01fdd7647 100644 --- a/tests/config/__snapshots__/default-config-staging.test.js.snap +++ b/tests/config/__snapshots__/default-config-staging.test.js.snap @@ -33,6 +33,6 @@ Object { "rdsSandboxId": "default", "sandboxId": "default", "workerNamespace": "worker-default", - "workerVersion": 3, + "workerVersion": 4, } `; From df6701fe89ebc6769e6e845ad0a4cba11062cc91 Mon Sep 17 00:00:00 2001 From: Alex Pickering Date: Wed, 29 Mar 2023 12:22:01 -0700 Subject: [PATCH 3/3] remove logs; fix lint Signed-off-by: Alex Pickering --- .../helpers/pipeline/getPipelineStatus.js | 4 +-- .../pipeline/pipelineConstruct/index.js | 6 ++++- .../helpers/pipeline/shouldPipelineRerun.js | 4 ++- ...20230117130009_add_last_pipeline_params.js | 25 +++++++------------ .../worker/submitEmbeddingWork.test.js | 2 +- .../worker/submitMarkerHeatmapWork.test.js | 2 +- 6 files changed, 21 insertions(+), 22 deletions(-) diff --git a/src/api.v2/helpers/pipeline/getPipelineStatus.js b/src/api.v2/helpers/pipeline/getPipelineStatus.js index 405e4b384..6eebb8015 100644 --- a/src/api.v2/helpers/pipeline/getPipelineStatus.js +++ b/src/api.v2/helpers/pipeline/getPipelineStatus.js @@ -37,7 +37,7 @@ const gem2sPipelineSteps = [ const seuratPipelineSteps = [ 'DownloadSeurat', 'ProcessSeurat', - 'UploadSeuratToAWS' + 'UploadSeuratToAWS', ]; // pipelineStepNames are the names of pipeline steps for which we @@ -292,7 +292,7 @@ const getPipelineStatus = async (experimentId, processName) => { let response = null; const { executionArn = null, stateMachineArn = null, lastStatusResponse } = pipelineExecution; - const shouldRerun = await shouldPipelineRerun(experimentId, processName); + const shouldRerun = await shouldPipelineRerun(experimentId, processName); try { execution = await stepFunctions.describeExecution({ diff --git a/src/api.v2/helpers/pipeline/pipelineConstruct/index.js b/src/api.v2/helpers/pipeline/pipelineConstruct/index.js index 5051d1399..416e7bc3c 100644 --- a/src/api.v2/helpers/pipeline/pipelineConstruct/index.js +++ b/src/api.v2/helpers/pipeline/pipelineConstruct/index.js @@ -12,8 +12,12 @@ const ExperimentExecution = require('../../../model/ExperimentExecution'); const getLogger = require('../../../../utils/getLogger'); const { - getGem2sPipelineSkeleton, getQcPipelineSkeleton, getSubsetPipelineSkeleton, getSeuratPipelineSkeleton, + getGem2sPipelineSkeleton, + getQcPipelineSkeleton, + getSubsetPipelineSkeleton, + getSeuratPipelineSkeleton, } = require('./skeletons'); + const { getQcStepsToRun, qcStepsWithFilterSettings } = require('./qcHelpers'); const needsBatchJob = require('../batch/needsBatchJob'); diff --git a/src/api.v2/helpers/pipeline/shouldPipelineRerun.js b/src/api.v2/helpers/pipeline/shouldPipelineRerun.js index 821d19342..564d81b80 100644 --- a/src/api.v2/helpers/pipeline/shouldPipelineRerun.js +++ b/src/api.v2/helpers/pipeline/shouldPipelineRerun.js @@ -90,7 +90,9 @@ const getPipelineParams = async (experimentId, rawSamples = undefined) => { }; const shouldPipelineRerun = async (experimentId, pipelineType) => { - const execution = await new ExperimentExecution().findOne({ experiment_id: experimentId, pipeline_type: pipelineType }); + const execution = await new ExperimentExecution() + .findOne({ experiment_id: experimentId, pipeline_type: pipelineType }); + if (execution === undefined) return true; const currentParams = await getPipelineParams(experimentId); diff --git a/src/sql/migrations/20230117130009_add_last_pipeline_params.js b/src/sql/migrations/20230117130009_add_last_pipeline_params.js index 30180cf8a..4a7fe71b7 100644 --- a/src/sql/migrations/20230117130009_add_last_pipeline_params.js +++ b/src/sql/migrations/20230117130009_add_last_pipeline_params.js @@ -1,23 +1,16 @@ -exports.up = (knex) => { - const hasLastGem2sParams = knex.schema.hasColumn('experiment_execution', 'last_gem2s_params'); - - knex.schema.hasColumn('experiment_execution', 'last_gem2s_params') - .then((exists) => { - console.log('exists'); - console.log(exists); - }); - - console.log('hasLastGem2sParams'); - console.log(hasLastGem2sParams); +exports.up = async (knex) => { + // for HMS deployment last_gem2s_params was always called last_pipeline_params + const hasLastGem2sParams = await knex.schema.hasColumn('experiment_execution', 'last_gem2s_params'); if (hasLastGem2sParams) { - console.log('here!!!'); - return knex.schema.alterTable('experiment_execution', (table) => { + await knex.schema.alterTable('experiment_execution', (table) => { table.renameColumn('last_gem2s_params', 'last_pipeline_params'); }); } }; -exports.down = (knex) => knex.schema.alterTable('experiment_execution', (table) => { - table.renameColumn('last_pipeline_params', 'last_gem2s_params'); -}); +exports.down = async (knex) => { + await knex.schema.alterTable('experiment_execution', (table) => { + table.renameColumn('last_pipeline_params', 'last_gem2s_params'); + }); +}; diff --git a/tests/api.v2/helpers/worker/submitEmbeddingWork.test.js b/tests/api.v2/helpers/worker/submitEmbeddingWork.test.js index 01e466d6f..1d73d017f 100644 --- a/tests/api.v2/helpers/worker/submitEmbeddingWork.test.js +++ b/tests/api.v2/helpers/worker/submitEmbeddingWork.test.js @@ -57,7 +57,7 @@ describe('submitWorkEmbedding', () => { expect(createObjectHash.mock.calls).toMatchSnapshot(); // this ETag should match exactly the one in // loadEmbedding.defaultParams.test.js - expect(ETag).toEqual('fb7a8a67c7806fa90c69b9cdb16dd478'); // pragma: allowlist secret + expect(ETag).toEqual('163301323ab0ee82902d1574a5dcd061'); // pragma: allowlist secret expect(validateAndSubmitWork).toBeCalledTimes(1); }); }); diff --git a/tests/api.v2/helpers/worker/submitMarkerHeatmapWork.test.js b/tests/api.v2/helpers/worker/submitMarkerHeatmapWork.test.js index 43926a4db..f09101dfc 100644 --- a/tests/api.v2/helpers/worker/submitMarkerHeatmapWork.test.js +++ b/tests/api.v2/helpers/worker/submitMarkerHeatmapWork.test.js @@ -57,7 +57,7 @@ describe('submitWorkEmbedding', () => { expect(createObjectHash.mock.calls).toMatchSnapshot(); // this ETag should match exactly the one in // loadMarkerGenes.defaultParams.test.js - expect(ETag).toEqual('a973357e8c6273f0f154a179b12c6d25'); // pragma: allowlist secret + expect(ETag).toEqual('7f2b831012fe0d6bebf8fcca49ccd057'); // pragma: allowlist secret expect(validateAndSubmitWork).toBeCalledTimes(1); });