From fc6cc884ec604867d691d20c13401c47b469bc1b Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Wed, 18 Dec 2024 01:32:17 +0400 Subject: [PATCH 1/7] feat(lib): collect child names to check regroup criteria --- src/if-run/lib/compute.ts | 51 ++++++++++++++++++++++++++------------- 1 file changed, 34 insertions(+), 17 deletions(-) diff --git a/src/if-run/lib/compute.ts b/src/if-run/lib/compute.ts index 0381fee1..74f03b2d 100644 --- a/src/if-run/lib/compute.ts +++ b/src/if-run/lib/compute.ts @@ -22,12 +22,15 @@ const { OBSERVING, } = STRINGS; +const childNames = new Set(); + /** * Traverses all child nodes based on children grouping. */ const traverse = async (children: any, params: ComputeParams) => { for (const child in children) { console.debug(COMPUTING_COMPONENT_PIPELINE(child)); + childNames.add(child); await computeNode(children[child], params); } }; @@ -129,6 +132,7 @@ const computeNode = async (node: Node, params: ComputeParams): Promise => { node.inputs = outputStorage; if (params.context.explainer) { + console.log('reached here'); addExplainData({ pluginName, metadata: plugin.metadata, @@ -143,27 +147,40 @@ const computeNode = async (node: Node, params: ComputeParams): Promise => { if ((noFlags || params.regroup) && pipelineCopy.regroup) { const originalOutputs = params.append ? node.outputs || [] : []; - node.children = Regroup( - outputStorage, - originalOutputs, - pipelineCopy.regroup + // Grabs all the values according to grouping criteria. + const regroupValues = pipelineCopy.regroup + .map(group => [...new Set(outputStorage.map(output => output[group]))]) + .flat(); + // Checks if regroup values are present in the children list. + const isRegrouped = regroupValues.every(one => + [...childNames].includes(one) ); + console.log(isRegrouped); + console.log(regroupValues); - delete node.inputs; - delete node.outputs; + if (!isRegrouped) { + node.children = Regroup( + outputStorage, + originalOutputs, + pipelineCopy.regroup + ); - debugLogger.setExecutingPluginName(); - console.debug(REGROUPING); + delete node.inputs; + delete node.outputs; - return traverse(node.children, { - ...params, - pipeline: { - ...pipelineCopy, - regroup: undefined, - }, - defaults, - config, - }); + debugLogger.setExecutingPluginName(); + console.debug(REGROUPING); + + return traverse(node.children, { + ...params, + pipeline: { + ...pipelineCopy, + regroup: undefined, + }, + defaults, + config, + }); + } } console.debug('\n'); From 91073dd1caeb17d091f4cd8e7ca8231624142c7f Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Wed, 18 Dec 2024 01:33:28 +0400 Subject: [PATCH 2/7] test(lib): update uni tests for compute --- src/__tests__/if-run/lib/compute.test.ts | 34 +++++++++++++++++++++--- 1 file changed, 31 insertions(+), 3 deletions(-) diff --git a/src/__tests__/if-run/lib/compute.test.ts b/src/__tests__/if-run/lib/compute.test.ts index a1d5f0a3..fec65fb2 100644 --- a/src/__tests__/if-run/lib/compute.test.ts +++ b/src/__tests__/if-run/lib/compute.test.ts @@ -22,13 +22,14 @@ jest.mock('../../../common/util/logger', () => ({ }, })); -import * as explainer from '../../../if-run/lib/explain'; - -import {compute} from '../../../if-run/lib/compute'; import {ComputeParams} from '../../../if-run/types/compute'; import {pluginStorage} from '../../../if-run/util/plugin-storage'; describe('lib/compute: ', () => { + beforeEach(() => { + jest.resetModules(); + }); + /** * Mock plugins. */ @@ -134,7 +135,12 @@ describe('lib/compute: ', () => { const paramsExecuteWithAppend = {...paramsExecute, append: true}; describe('compute(): ', () => { + beforeEach(() => { + jest.resetModules(); + }); + it('computes simple tree with execute plugin.', async () => { + const {compute} = require('../../../if-run/lib/compute'); const tree = { children: { mockChild: { @@ -156,6 +162,7 @@ describe('lib/compute: ', () => { }); it('computes simple tree with regroup on inputs only (no compute).', async () => { + const {compute} = require('../../../if-run/lib/compute'); const tree = { children: { mockChild: { @@ -185,6 +192,7 @@ describe('lib/compute: ', () => { }); it('computes simple tree with regroup, grouping inputs and outputs.', async () => { + const {compute} = require('../../../if-run/lib/compute'); const tree = { children: { mockChild: { @@ -232,6 +240,7 @@ describe('lib/compute: ', () => { }); it('computes simple tree with defaults and execute plugin.', async () => { + const {compute} = require('../../../if-run/lib/compute'); const tree = { children: { mockChild: { @@ -259,6 +268,7 @@ describe('lib/compute: ', () => { }); it('computes nested tree with defaults and execute plugin.', async () => { + const {compute} = require('../../../if-run/lib/compute'); const tree = { children: { mockChild1: { @@ -308,6 +318,7 @@ describe('lib/compute: ', () => { }); it('computes simple tree with no defaults and no inputs with execute plugin.', async () => { + const {compute} = require('../../../if-run/lib/compute'); const tree = { children: { mockChild: { @@ -323,6 +334,7 @@ describe('lib/compute: ', () => { }); it('computes simple tree with defaults and no inputs with execute plugin.', async () => { + const {compute} = require('../../../if-run/lib/compute'); const tree = { children: { mockChild: { @@ -343,6 +355,7 @@ describe('lib/compute: ', () => { }); it('computes simple tree with append, preserving existing outputs.', async () => { + const {compute} = require('../../../if-run/lib/compute'); const tree = { children: { mockChild: { @@ -378,6 +391,7 @@ describe('lib/compute: ', () => { }); it('computes simple tree with append when outputs is null.', async () => { + const {compute} = require('../../../if-run/lib/compute'); const tree = { children: { mockChild: { @@ -402,6 +416,7 @@ describe('lib/compute: ', () => { }); it('computes simple tree with regroup and append, with existing outputs preserved and regrouped without re-computing.', async () => { + const {compute} = require('../../../if-run/lib/compute'); const tree = { children: { mockChild: { @@ -434,6 +449,7 @@ describe('lib/compute: ', () => { }); it('computes simple tree with regroup and append, with existing outputs preserved and without new outputs.', async () => { + const {compute} = require('../../../if-run/lib/compute'); const tree = { children: { mockChild: { @@ -460,6 +476,7 @@ describe('lib/compute: ', () => { }); it('computes simple tree with regroup and no append, with existing outputs that are removed.', async () => { + const {compute} = require('../../../if-run/lib/compute'); const tree = { children: { mockChild: { @@ -488,6 +505,7 @@ describe('lib/compute: ', () => { }); it('computes simple tree with observe plugin.', async () => { + const {compute} = require('../../../if-run/lib/compute'); const tree = { children: { mockChild: { @@ -507,6 +525,7 @@ describe('lib/compute: ', () => { }); it('computes simple tree with observe plugin.', async () => { + const {compute} = require('../../../if-run/lib/compute'); const tree = { children: { mockChild: { @@ -526,6 +545,7 @@ describe('lib/compute: ', () => { }); it('observes simple tree with observe plugin.', async () => { + const {compute} = require('../../../if-run/lib/compute'); const tree = { children: { mockChild: { @@ -545,6 +565,7 @@ describe('lib/compute: ', () => { }); it('observes simple tree with config.', async () => { + const {compute} = require('../../../if-run/lib/compute'); const tree = { children: { mockChild: { @@ -565,6 +586,7 @@ describe('lib/compute: ', () => { }); it('warns when pipeline is null.', async () => { + const {compute} = require('../../../if-run/lib/compute'); process.env.LOGGER = 'invalid'; const tree = { children: { @@ -583,6 +605,7 @@ describe('lib/compute: ', () => { }); it('warns when pipeline is an empty object.', async () => { + const {compute} = require('../../../if-run/lib/compute'); const tree = { children: { mockChild: { @@ -598,6 +621,7 @@ describe('lib/compute: ', () => { }); it('warns when config is provided in the tree.', async () => { + const {compute} = require('../../../if-run/lib/compute'); process.env.LOGGER = 'true'; const tree = { children: { @@ -624,6 +648,7 @@ describe('lib/compute: ', () => { }); it('warns when config is provided in the tree and it is null.', async () => { + const {compute} = require('../../../if-run/lib/compute'); process.env.LOGGER = 'empty'; const tree = { children: { @@ -650,6 +675,8 @@ describe('lib/compute: ', () => { }); it('observes simple tree with execute plugin and explain property.', async () => { + const {compute} = require('../../../if-run/lib/compute'); + const explainer = require('../../../if-run/lib/explain'); const tree = { children: { mockChild: { @@ -678,6 +705,7 @@ describe('lib/compute: ', () => { }); it('computes simple tree with execute plugin and explain property.', async () => { + const {compute} = require('../../../if-run/lib/compute'); const tree = { children: { mockChild: { From b576e7aaa08050fd30498e569f189fa0cf44e50f Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Wed, 18 Dec 2024 01:35:08 +0400 Subject: [PATCH 3/7] test(lib): drop second before each hook --- src/__tests__/if-run/lib/compute.test.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/__tests__/if-run/lib/compute.test.ts b/src/__tests__/if-run/lib/compute.test.ts index fec65fb2..5718ba88 100644 --- a/src/__tests__/if-run/lib/compute.test.ts +++ b/src/__tests__/if-run/lib/compute.test.ts @@ -135,10 +135,6 @@ describe('lib/compute: ', () => { const paramsExecuteWithAppend = {...paramsExecute, append: true}; describe('compute(): ', () => { - beforeEach(() => { - jest.resetModules(); - }); - it('computes simple tree with execute plugin.', async () => { const {compute} = require('../../../if-run/lib/compute'); const tree = { From 90df77a37c3a70b5fd84be851ca8fd0022e29713 Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Wed, 18 Dec 2024 01:36:12 +0400 Subject: [PATCH 4/7] fix(lib): drop unnecessary logs --- src/if-run/lib/compute.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/if-run/lib/compute.ts b/src/if-run/lib/compute.ts index 74f03b2d..7038857f 100644 --- a/src/if-run/lib/compute.ts +++ b/src/if-run/lib/compute.ts @@ -132,7 +132,6 @@ const computeNode = async (node: Node, params: ComputeParams): Promise => { node.inputs = outputStorage; if (params.context.explainer) { - console.log('reached here'); addExplainData({ pluginName, metadata: plugin.metadata, @@ -155,8 +154,6 @@ const computeNode = async (node: Node, params: ComputeParams): Promise => { const isRegrouped = regroupValues.every(one => [...childNames].includes(one) ); - console.log(isRegrouped); - console.log(regroupValues); if (!isRegrouped) { node.children = Regroup( From 4f7a3164f07923e6fddf9b56a8f155d24c2a27c1 Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Wed, 18 Dec 2024 01:45:55 +0400 Subject: [PATCH 5/7] test(lib): add additional case to cover already regrouped tree --- src/__tests__/if-run/lib/compute.test.ts | 26 ++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/src/__tests__/if-run/lib/compute.test.ts b/src/__tests__/if-run/lib/compute.test.ts index 5718ba88..c0261ab2 100644 --- a/src/__tests__/if-run/lib/compute.test.ts +++ b/src/__tests__/if-run/lib/compute.test.ts @@ -187,6 +187,32 @@ describe('lib/compute: ', () => { expect(response.children.mockChild.children).toEqual(expectedResponse); }); + it('skips regrouping on already regrouped data.', async () => { + const {compute} = require('../../../if-run/lib/compute'); + const expectedResponse = { + 'uk-west': { + inputs: [{region: 'uk-west', timestamp: 'mock-timestamp-1'}], + }, + 'uk-east': { + inputs: [ + {region: 'uk-east', timestamp: 'mock-timestamp-2'}, + {region: 'uk-east', timestamp: 'mock-timestamp-3'}, + ], + }, + }; + const tree = { + children: { + mockChild: { + pipeline: {regroup: ['region']}, + children: expectedResponse, + }, + }, + }; + const response = await compute(tree, paramsExecute); + + expect(response.children.mockChild.children).toEqual(expectedResponse); + }); + it('computes simple tree with regroup, grouping inputs and outputs.', async () => { const {compute} = require('../../../if-run/lib/compute'); const tree = { From 57c29556f8f61c435ed75e08e940b39db4fe9801 Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Wed, 18 Dec 2024 18:05:34 +0400 Subject: [PATCH 6/7] feat(config): add skipping regroup message --- src/if-run/config/strings.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/if-run/config/strings.ts b/src/if-run/config/strings.ts index 41b9dc01..951f98a7 100644 --- a/src/if-run/config/strings.ts +++ b/src/if-run/config/strings.ts @@ -51,6 +51,7 @@ Note that for the '--output' option you also need to define the output type in y COMPUTING_COMPONENT_PIPELINE: (component: string) => `**Computing \`${component}\` pipeline**`, REGROUPING: 'Regrouping', + SKIPPING_REGROUP: 'Already correctly grouped - skipping regrouping.', OBSERVING: (nodeName: string) => `Running observe pipeline: \`${nodeName}\` plugin`, MERGING_DEFAULTS_WITH_INPUT_DATA: 'Merging defaults with input data', From 43db5f3229826f265dd60a848dde2bb209712a5d Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Wed, 18 Dec 2024 18:08:24 +0400 Subject: [PATCH 7/7] feat(lib): add skipping regroup message to compute --- src/if-run/lib/compute.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/if-run/lib/compute.ts b/src/if-run/lib/compute.ts index 7038857f..f77b1b99 100644 --- a/src/if-run/lib/compute.ts +++ b/src/if-run/lib/compute.ts @@ -20,6 +20,7 @@ const { COMPUTING_COMPONENT_PIPELINE, REGROUPING, OBSERVING, + SKIPPING_REGROUP, } = STRINGS; const childNames = new Set(); @@ -177,6 +178,8 @@ const computeNode = async (node: Node, params: ComputeParams): Promise => { defaults, config, }); + } else { + console.debug(SKIPPING_REGROUP); } }