Skip to content

Commit

Permalink
Merge pull request #1108 from Green-Software-Foundation/regroup
Browse files Browse the repository at this point in the history 
Skip regrouping on already regrouped manifests
  • Loading branch information
@jmcook1186
jmcook1186 authored Dec 18, 2024
2 parents 8eac74f + 43db5f3 commit d63b807
Show file tree
Hide file tree
Showing 3 changed files with 88 additions and 20 deletions.
56 changes: 53 additions & 3 deletions src/__tests__/if-run/lib/compute.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,13 +22,14 @@ jest.mock('../../../common/util/logger', () => ({
},
}));

import * as explainer from '../../../if-run/lib/explain';

import {compute} from '../../../if-run/lib/compute';
import {ComputeParams} from '../../../if-run/types/compute';
import {pluginStorage} from '../../../if-run/util/plugin-storage';

describe('lib/compute: ', () => {
beforeEach(() => {
jest.resetModules();
});

/**
* Mock plugins.
*/
Expand Down Expand Up @@ -135,6 +136,7 @@ describe('lib/compute: ', () => {

describe('compute(): ', () => {
it('computes simple tree with execute plugin.', async () => {
const {compute} = require('../../../if-run/lib/compute');
const tree = {
children: {
mockChild: {
Expand All @@ -156,6 +158,7 @@ describe('lib/compute: ', () => {
});

it('computes simple tree with regroup on inputs only (no compute).', async () => {
const {compute} = require('../../../if-run/lib/compute');
const tree = {
children: {
mockChild: {
Expand Down Expand Up @@ -184,7 +187,34 @@ describe('lib/compute: ', () => {
expect(response.children.mockChild.children).toEqual(expectedResponse);
});

it('skips regrouping on already regrouped data.', async () => {
const {compute} = require('../../../if-run/lib/compute');
const expectedResponse = {
'uk-west': {
inputs: [{region: 'uk-west', timestamp: 'mock-timestamp-1'}],
},
'uk-east': {
inputs: [
{region: 'uk-east', timestamp: 'mock-timestamp-2'},
{region: 'uk-east', timestamp: 'mock-timestamp-3'},
],
},
};
const tree = {
children: {
mockChild: {
pipeline: {regroup: ['region']},
children: expectedResponse,
},
},
};
const response = await compute(tree, paramsExecute);

expect(response.children.mockChild.children).toEqual(expectedResponse);
});

it('computes simple tree with regroup, grouping inputs and outputs.', async () => {
const {compute} = require('../../../if-run/lib/compute');
const tree = {
children: {
mockChild: {
Expand Down Expand Up @@ -232,6 +262,7 @@ describe('lib/compute: ', () => {
});

it('computes simple tree with defaults and execute plugin.', async () => {
const {compute} = require('../../../if-run/lib/compute');
const tree = {
children: {
mockChild: {
Expand Down Expand Up @@ -259,6 +290,7 @@ describe('lib/compute: ', () => {
});

it('computes nested tree with defaults and execute plugin.', async () => {
const {compute} = require('../../../if-run/lib/compute');
const tree = {
children: {
mockChild1: {
Expand Down Expand Up @@ -308,6 +340,7 @@ describe('lib/compute: ', () => {
});

it('computes simple tree with no defaults and no inputs with execute plugin.', async () => {
const {compute} = require('../../../if-run/lib/compute');
const tree = {
children: {
mockChild: {
Expand All @@ -323,6 +356,7 @@ describe('lib/compute: ', () => {
});

it('computes simple tree with defaults and no inputs with execute plugin.', async () => {
const {compute} = require('../../../if-run/lib/compute');
const tree = {
children: {
mockChild: {
Expand All @@ -343,6 +377,7 @@ describe('lib/compute: ', () => {
});

it('computes simple tree with append, preserving existing outputs.', async () => {
const {compute} = require('../../../if-run/lib/compute');
const tree = {
children: {
mockChild: {
Expand Down Expand Up @@ -378,6 +413,7 @@ describe('lib/compute: ', () => {
});

it('computes simple tree with append when outputs is null.', async () => {
const {compute} = require('../../../if-run/lib/compute');
const tree = {
children: {
mockChild: {
Expand All @@ -402,6 +438,7 @@ describe('lib/compute: ', () => {
});

it('computes simple tree with regroup and append, with existing outputs preserved and regrouped without re-computing.', async () => {
const {compute} = require('../../../if-run/lib/compute');
const tree = {
children: {
mockChild: {
Expand Down Expand Up @@ -434,6 +471,7 @@ describe('lib/compute: ', () => {
});

it('computes simple tree with regroup and append, with existing outputs preserved and without new outputs.', async () => {
const {compute} = require('../../../if-run/lib/compute');
const tree = {
children: {
mockChild: {
Expand All @@ -460,6 +498,7 @@ describe('lib/compute: ', () => {
});

it('computes simple tree with regroup and no append, with existing outputs that are removed.', async () => {
const {compute} = require('../../../if-run/lib/compute');
const tree = {
children: {
mockChild: {
Expand Down Expand Up @@ -488,6 +527,7 @@ describe('lib/compute: ', () => {
});

it('computes simple tree with observe plugin.', async () => {
const {compute} = require('../../../if-run/lib/compute');
const tree = {
children: {
mockChild: {
Expand All @@ -507,6 +547,7 @@ describe('lib/compute: ', () => {
});

it('computes simple tree with observe plugin.', async () => {
const {compute} = require('../../../if-run/lib/compute');
const tree = {
children: {
mockChild: {
Expand All @@ -526,6 +567,7 @@ describe('lib/compute: ', () => {
});

it('observes simple tree with observe plugin.', async () => {
const {compute} = require('../../../if-run/lib/compute');
const tree = {
children: {
mockChild: {
Expand All @@ -545,6 +587,7 @@ describe('lib/compute: ', () => {
});

it('observes simple tree with config.', async () => {
const {compute} = require('../../../if-run/lib/compute');
const tree = {
children: {
mockChild: {
Expand All @@ -565,6 +608,7 @@ describe('lib/compute: ', () => {
});

it('warns when pipeline is null.', async () => {
const {compute} = require('../../../if-run/lib/compute');
process.env.LOGGER = 'invalid';
const tree = {
children: {
Expand All @@ -583,6 +627,7 @@ describe('lib/compute: ', () => {
});

it('warns when pipeline is an empty object.', async () => {
const {compute} = require('../../../if-run/lib/compute');
const tree = {
children: {
mockChild: {
Expand All @@ -598,6 +643,7 @@ describe('lib/compute: ', () => {
});

it('warns when config is provided in the tree.', async () => {
const {compute} = require('../../../if-run/lib/compute');
process.env.LOGGER = 'true';
const tree = {
children: {
Expand All @@ -624,6 +670,7 @@ describe('lib/compute: ', () => {
});

it('warns when config is provided in the tree and it is null.', async () => {
const {compute} = require('../../../if-run/lib/compute');
process.env.LOGGER = 'empty';
const tree = {
children: {
Expand All @@ -650,6 +697,8 @@ describe('lib/compute: ', () => {
});

it('observes simple tree with execute plugin and explain property.', async () => {
const {compute} = require('../../../if-run/lib/compute');
const explainer = require('../../../if-run/lib/explain');
const tree = {
children: {
mockChild: {
Expand Down Expand Up @@ -678,6 +727,7 @@ describe('lib/compute: ', () => {
});

it('computes simple tree with execute plugin and explain property.', async () => {
const {compute} = require('../../../if-run/lib/compute');
const tree = {
children: {
mockChild: {
Expand Down
1 change: 1 addition & 0 deletions src/if-run/config/strings.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ Note that for the '--output' option you also need to define the output type in y
COMPUTING_COMPONENT_PIPELINE: (component: string) =>
`**Computing \`${component}\` pipeline**`,
REGROUPING: 'Regrouping',
SKIPPING_REGROUP: 'Already correctly grouped - skipping regrouping.',
OBSERVING: (nodeName: string) =>
`Running observe pipeline: \`${nodeName}\` plugin`,
MERGING_DEFAULTS_WITH_INPUT_DATA: 'Merging defaults with input data',
Expand Down
51 changes: 34 additions & 17 deletions src/if-run/lib/compute.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,14 +20,18 @@ const {
COMPUTING_COMPONENT_PIPELINE,
REGROUPING,
OBSERVING,
SKIPPING_REGROUP,
} = STRINGS;

const childNames = new Set();

/**
* Traverses all child nodes based on children grouping.
*/
const traverse = async (children: any, params: ComputeParams) => {
for (const child in children) {
console.debug(COMPUTING_COMPONENT_PIPELINE(child));
childNames.add(child);
await computeNode(children[child], params);
}
};
Expand Down Expand Up @@ -143,27 +147,40 @@ const computeNode = async (node: Node, params: ComputeParams): Promise<any> => {
if ((noFlags || params.regroup) && pipelineCopy.regroup) {
const originalOutputs = params.append ? node.outputs || [] : [];

node.children = Regroup(
outputStorage,
originalOutputs,
pipelineCopy.regroup
// Grabs all the values according to grouping criteria.
const regroupValues = pipelineCopy.regroup
.map(group => [...new Set(outputStorage.map(output => output[group]))])
.flat();
// Checks if regroup values are present in the children list.
const isRegrouped = regroupValues.every(one =>
[...childNames].includes(one)
);

delete node.inputs;
delete node.outputs;
if (!isRegrouped) {
node.children = Regroup(
outputStorage,
originalOutputs,
pipelineCopy.regroup
);

debugLogger.setExecutingPluginName();
console.debug(REGROUPING);
delete node.inputs;
delete node.outputs;

return traverse(node.children, {
...params,
pipeline: {
...pipelineCopy,
regroup: undefined,
},
defaults,
config,
});
debugLogger.setExecutingPluginName();
console.debug(REGROUPING);

return traverse(node.children, {
...params,
pipeline: {
...pipelineCopy,
regroup: undefined,
},
defaults,
config,
});
} else {
console.debug(SKIPPING_REGROUP);
}
}

console.debug('\n');
Expand Down

0 comments on commit d63b807

Please sign in to comment.