Skip to content

Commit

Permalink
Merge pull request #45 from RIKEN-RCCS/fixEnvironmentVariable
Browse files Browse the repository at this point in the history 
remove WHEEL attached environment variables before dispatching task
  • Loading branch information
@so5
so5 authored Apr 8, 2024
2 parents d3797ba + f63558e commit 86f8a64
Show file tree
Hide file tree
Showing 5 changed files with 121 additions and 18 deletions.
31 changes: 23 additions & 8 deletions server/app/core/dispatcher.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,17 @@ const {
foreachKeepLoopInstance
} = require("./loopUtils.js");
const { makeCmd } = require("./psUtils.js");
const wheelSystemEnv=[
"WHEEL_CURRENT_INDEX",
"WHEEL_NEXT_INDEX",
"WHEEL_PREV_INDEX",
"WHEEL_FOR_START",
"WHEEL_FOR_END",
"WHEEL_FOR_STEP",
"WHEEL_PS_PARAM",
"WHEEL_FOREACH_LEN",
"WHEEL_REMOTE_WORK"
]

const taskDB = new Map();

Expand Down Expand Up @@ -173,7 +184,7 @@ class Dispatcher extends EventEmitter {

if (this.firstCall) {
await this._asyncInit();
const childComponents = await getChildren(this.projectRootDir, this.cwfID);
const childComponents = await getChildren(this.projectRootDir, this.cwfDir, true);
this.currentSearchList = childComponents.filter((component)=>{
return isInitialComponent(component);
});
Expand All @@ -182,7 +193,6 @@ class Dispatcher extends EventEmitter {
}));
this.firstCall = false;
}

this.logger.trace("currentList:", this.currentSearchList.map((e)=>{
return e.name;
}));
Expand Down Expand Up @@ -404,8 +414,9 @@ class Dispatcher extends EventEmitter {
return e.ID === id
});
});
const nextComponents = await Promise.all(nextComponentIDs.map((id)=>{
return this._getComponent(id);
const nextComponents = await Promise.all(nextComponentIDs.map(async (id)=>{
const tmp = await this._getComponent(id);
return tmp
}));

Array.prototype.push.apply(this.pendingComponents, nextComponents);
Expand Down Expand Up @@ -433,7 +444,7 @@ class Dispatcher extends EventEmitter {
}

async _dispatchTask(task) {
//this.logger.debug("_dispatchTask called", task.name);
this.logger.trace("_dispatchTask called", task.name);
task.dispatchedTime = getDateString(true, true);
task.startTime = "not started"; //to be assigned in executer
task.endTime = "not finished"; //to be assigned in executer
Expand Down Expand Up @@ -466,6 +477,9 @@ class Dispatcher extends EventEmitter {
if (task.usePSSettingFile === true) {
await this._bulkjobHandler(task);
}
wheelSystemEnv.forEach((envname)=>{
delete task.env[envname]
});
task.env = Object.assign(this.env, task.env);
task.parentType = this.cwfJson.type;

Expand Down Expand Up @@ -569,7 +583,7 @@ class Dispatcher extends EventEmitter {
//send back itself to searchList for next loop trip
this.pendingComponents.push(component);

const newComponent = Object.assign({}, component);
const newComponent = structuredClone(component);
newComponent.name = `${component.originalName}_${sanitizePath(component.currentIndex)}`;
newComponent.subComponent = true;

Expand All @@ -588,7 +602,8 @@ class Dispatcher extends EventEmitter {
newComponent.env.WHEEL_PREV_INDEX = component.currentIndex - step;
}
}
newComponent.env.WHEEL_NEXT_INDEX = getNextIndex(component);
const nextIndex = getNextIndex(component)
newComponent.env.WHEEL_NEXT_INDEX = nextIndex !== null ? nextIndex : "";
const dstDir = path.resolve(this.cwfDir, newComponent.name);

try {
Expand Down Expand Up @@ -739,7 +754,7 @@ class Dispatcher extends EventEmitter {
this.logger.debug("rewrite target files");
await rewriteTargetFile(templateRoot, instanceRoot, targetFiles, params);

const newComponent = Object.assign({}, component);
const newComponent = structuredClone(component);
newComponent.name = newName;
newComponent.subComponent = true;
const newComponentFilename = path.join(instanceRoot, componentJsonFilename);
Expand Down
2 changes: 1 addition & 1 deletion server/app/core/loopUtils.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ function foreachGetNextIndex(component) {
return e === component.currentIndex;
});

if (i === -1 || i === component.indexList.length - 1) {
if (i === -1 || i === component.indexList.length - 1 ) {
return null;
}
return component.indexList[i + 1];
Expand Down
11 changes: 9 additions & 2 deletions server/app/core/projectFilesOperator.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1043,8 +1043,15 @@ async function makeDir(basename, argSuffix) {
return dirname;
}

async function getChildren(projectRootDir, parentID) {
const dir = parentID === null ? projectRootDir : await getComponentDir(projectRootDir, parentID, true);
/**
* get array of child components
* @param {string} projectRootDir - project root directory path
* @param {string} parentID - parent component's ID or directory path
* @param {Boolean} isParentDir - if true, parentID is regard as path to parent directory, not ID string
* @returns {Component[]} - array of child components except for subComponent
*/
async function getChildren(projectRootDir, parentID, isParentDir) {
const dir = isParentDir ? parentID : parentID === null ? projectRootDir : await getComponentDir(projectRootDir, parentID, true);

if (!dir) {
return [];
Expand Down
2 changes: 1 addition & 1 deletion server/app/db/version.json
View file
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"version": "2024-0314-155703" }
{"version": "2024-0408-234448" }
93 changes: 87 additions & 6 deletions server/test/environmentVariable.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,14 +62,26 @@ describe("UT for environment variables", function() {
await updateComponent(projectRootDir, forTask.ID, "script", scriptName);
await fs.outputFile(path.join(projectRootDir, "for0", "task0", scriptName), scriptEcho);

const innerFor = await createNewComponent(projectRootDir, path.join(projectRootDir, "for0"), "for", { x: 10, y: 10 });
await updateComponent(projectRootDir, innerFor.ID, "start", 5);
await updateComponent(projectRootDir, innerFor.ID, "end", 1);
await updateComponent(projectRootDir, innerFor.ID, "step", -2);
const innerForTask = await createNewComponent(projectRootDir, path.join(projectRootDir, "for0", "for0"), "task", { x: 10, y: 10 });
await updateComponent(projectRootDir, innerForTask.ID, "script", scriptName);
const forUnderFor = await createNewComponent(projectRootDir, path.join(projectRootDir, "for0"), "for", { x: 10, y: 10 });
await updateComponent(projectRootDir, forUnderFor.ID, "start", 5);
await updateComponent(projectRootDir, forUnderFor.ID, "end", 1);
await updateComponent(projectRootDir, forUnderFor.ID, "step", -2);
const forUnderForTask = await createNewComponent(projectRootDir, path.join(projectRootDir, "for0", "for0"), "task", { x: 10, y: 10 });
await updateComponent(projectRootDir, forUnderForTask.ID, "script", scriptName);
await fs.outputFile(path.join(projectRootDir, "for0", "for0", "task0", scriptName), scriptEcho);

const whileUnderFor = await createNewComponent(projectRootDir, path.join(projectRootDir,"for0"), "while", { x: 11, y: 11 });
await updateComponent(projectRootDir, whileUnderFor.ID, "condition", "WHEEL_CURRENT_INDEX < 2");
const whileUnderForTask = await createNewComponent(projectRootDir, path.join(projectRootDir, "for0", "while0"), "task", { x: 10, y: 10 });
await updateComponent(projectRootDir, whileUnderForTask.ID, "script", scriptName);
await fs.outputFile(path.join(projectRootDir, "for0", "while0", "task0", scriptName), scriptEcho);

const foreachUnderFor = await createNewComponent(projectRootDir, path.join(projectRootDir,"for0"), "foreach", { x: 11, y: 11 });
await updateComponent(projectRootDir, foreachUnderFor.ID, "indexList", ["foo", "bar"]);
const foreachUnderForTask = await createNewComponent(projectRootDir, path.join(projectRootDir, "for0", "foreach0"), "task", { x: 10, y: 10 });
await updateComponent(projectRootDir, foreachUnderForTask.ID, "script", scriptName);
await fs.outputFile(path.join(projectRootDir, "for0", "foreach0", "task0", scriptName), scriptEcho);

const while0 = await createNewComponent(projectRootDir, projectRootDir, "while", { x: 11, y: 11 });
await updateComponent(projectRootDir, while0.ID, "condition", "WHEEL_CURRENT_INDEX < 2");
const whileTask = await createNewComponent(projectRootDir, path.join(projectRootDir, "while0"), "task", { x: 10, y: 10 });
Expand Down Expand Up @@ -108,6 +120,27 @@ describe("UT for environment variables", function() {
await updateComponent(projectRootDir, task0.ID, "script", scriptName);
await fs.outputFile(path.join(projectRootDir, "task0", scriptName), scriptEcho);

const forUnderPS = await createNewComponent(projectRootDir, path.join(projectRootDir, ps0.name), "for", { x: 10, y: 10 });
await updateComponent(projectRootDir, forUnderPS.ID, "start", 5);
await updateComponent(projectRootDir, forUnderPS.ID, "end", 1);
await updateComponent(projectRootDir, forUnderPS.ID, "step", -2);
const forUnderPSTask = await createNewComponent(projectRootDir, path.join(projectRootDir, ps0.name, "for0"), "task", { x: 10, y: 10 });
await updateComponent(projectRootDir, forUnderPSTask.ID, "script", scriptName);
await fs.outputFile(path.join(projectRootDir, ps0.name, "for0", "task0", scriptName), scriptEcho);

const whileUnderPS = await createNewComponent(projectRootDir, path.join(projectRootDir,ps0.name), "while", { x: 11, y: 11 });
await updateComponent(projectRootDir, whileUnderPS.ID, "condition", "WHEEL_CURRENT_INDEX < 2");
const whileUnderPSTask = await createNewComponent(projectRootDir, path.join(projectRootDir, ps0.name, "while0"), "task", { x: 10, y: 10 });
await updateComponent(projectRootDir, whileUnderPSTask.ID, "script", scriptName);
await fs.outputFile(path.join(projectRootDir, ps0.name, "while0", "task0", scriptName), scriptEcho);

const foreachUnderPS = await createNewComponent(projectRootDir, path.join(projectRootDir,ps0.name), "foreach", { x: 11, y: 11 });
await updateComponent(projectRootDir, foreachUnderPS.ID, "indexList", ["foo", "bar"]);
const foreachUnderPSTask = await createNewComponent(projectRootDir, path.join(projectRootDir, ps0.name, "foreach0"), "task", { x: 10, y: 10 });
await updateComponent(projectRootDir, foreachUnderPSTask.ID, "script", scriptName);
await fs.outputFile(path.join(projectRootDir, ps0.name, "foreach0", "task0", scriptName), scriptEcho);


state = await runProject(projectRootDir);
});
after(async()=>{
Expand Down Expand Up @@ -137,4 +170,52 @@ describe("UT for environment variables", function() {
expect(path.join(projectRootDir, "for0_2", "task0", logfilename)).to.be.a.file().with.contents.that.match(/^WHEEL_NEXT_INDEX=4$/m);
expect(path.join(projectRootDir, "for0_2", "task0", logfilename)).to.be.a.file().with.contents.that.match(/^WHEEL_PREV_INDEX=0$/m);
});
it("should have WHEEL_CURRENT_INDEX , WHEEL_PREV_INDEX, WHEEL_NEXT_INDEX, WHEEL_FOR_START, WHEEL_FOR_END, and WHEEL_FOR_STEP in task under inner for component", ()=>{
expect(path.join(projectRootDir, "for0_2","for0_3", "task0", logfilename)).to.be.a.file().with.contents.that.match(/^WHEEL_FOR_START=5$/m);
expect(path.join(projectRootDir, "for0_2","for0_3", "task0", logfilename)).to.be.a.file().with.contents.that.match(/^WHEEL_FOR_END=1$/m);
expect(path.join(projectRootDir, "for0_2","for0_3", "task0", logfilename)).to.be.a.file().with.contents.that.match(/^WHEEL_FOR_STEP=-2$/m);
expect(path.join(projectRootDir, "for0_2","for0_3", "task0", logfilename)).to.be.a.file().with.contents.that.match(/^WHEEL_CURRENT_INDEX=3$/m);
expect(path.join(projectRootDir, "for0_2","for0_3", "task0", logfilename)).to.be.a.file().with.contents.that.match(/^WHEEL_NEXT_INDEX=1$/m);
expect(path.join(projectRootDir, "for0_2","for0_3", "task0", logfilename)).to.be.a.file().with.contents.that.match(/^WHEEL_PREV_INDEX=5$/m);
});
it("should have WHEEL_CURRENT_INDEX , WHEEL_PREV_INDEX, WHEEL_NEXT_INDEX, WHEEL_FOR_START, WHEEL_FOR_END, and WHEEL_FOR_STEP in task under inner while component", ()=>{
expect(path.join(projectRootDir, "for0_2","while0_1", "task0", logfilename)).to.be.a.file().with.contents.that.match(/^WHEEL_FOR_START=0$/m);
expect(path.join(projectRootDir, "for0_2","while0_1", "task0", logfilename)).to.be.a.file().with.contents.that.match(/^WHEEL_FOR_END=3$/m);
expect(path.join(projectRootDir, "for0_2","while0_1", "task0", logfilename)).to.be.a.file().with.contents.that.match(/^WHEEL_FOR_STEP=2$/m);
expect(path.join(projectRootDir, "for0_2","while0_1", "task0", logfilename)).to.be.a.file().with.contents.that.match(/^WHEEL_CURRENT_INDEX=1$/m);
expect(path.join(projectRootDir, "for0_2","while0_1", "task0", logfilename)).to.be.a.file().with.contents.that.match(/^WHEEL_NEXT_INDEX=2$/m);
expect(path.join(projectRootDir, "for0_2","while0_1", "task0", logfilename)).to.be.a.file().with.contents.that.match(/^WHEEL_PREV_INDEX=0$/m);
});
it("should have WHEEL_CURRENT_INDEX , WHEEL_PREV_INDEX, WHEEL_NEXT_INDEX, WHEEL_FOR_START, WHEEL_FOR_END, and WHEEL_FOR_STEP in task under inner foreach component", ()=>{
expect(path.join(projectRootDir, "for0_2","foreach0_bar", "task0", logfilename)).to.be.a.file().with.contents.that.match(/^WHEEL_FOR_START=0$/m);
expect(path.join(projectRootDir, "for0_2","foreach0_bar", "task0", logfilename)).to.be.a.file().with.contents.that.match(/^WHEEL_FOR_END=3$/m);
expect(path.join(projectRootDir, "for0_2","foreach0_bar", "task0", logfilename)).to.be.a.file().with.contents.that.match(/^WHEEL_FOR_STEP=2$/m);
expect(path.join(projectRootDir, "for0_2","foreach0_bar", "task0", logfilename)).to.be.a.file().with.contents.that.match(/^WHEEL_CURRENT_INDEX=bar$/m);
expect(path.join(projectRootDir, "for0_2","foreach0_bar", "task0", logfilename)).to.be.a.file().with.contents.that.match(/^WHEEL_NEXT_INDEX=$/m);
expect(path.join(projectRootDir, "for0_2","foreach0_bar", "task0", logfilename)).to.be.a.file().with.contents.that.match(/^WHEEL_PREV_INDEX=foo$/m);
});
it("should have WHEEL_CURRENT_INDEX , WHEEL_PREV_INDEX, WHEEL_NEXT_INDEX, WHEEL_FOR_START, WHEEL_FOR_END, and WHEEL_FOR_STEP in task under inner for component", ()=>{
expect(path.join(projectRootDir, "PS0_KEYWORD1_2","for0_3", "task0", logfilename)).to.be.a.file().with.contents.that.match(/^WHEEL_FOR_START=5$/m);
expect(path.join(projectRootDir, "PS0_KEYWORD1_2","for0_3", "task0", logfilename)).to.be.a.file().with.contents.that.match(/^WHEEL_FOR_END=1$/m);
expect(path.join(projectRootDir, "PS0_KEYWORD1_2","for0_3", "task0", logfilename)).to.be.a.file().with.contents.that.match(/^WHEEL_FOR_STEP=-2$/m);
expect(path.join(projectRootDir, "PS0_KEYWORD1_2","for0_3", "task0", logfilename)).to.be.a.file().with.contents.that.match(/^WHEEL_CURRENT_INDEX=3$/m);
expect(path.join(projectRootDir, "PS0_KEYWORD1_2","for0_3", "task0", logfilename)).to.be.a.file().with.contents.that.match(/^WHEEL_NEXT_INDEX=1$/m);
expect(path.join(projectRootDir, "PS0_KEYWORD1_2","for0_3", "task0", logfilename)).to.be.a.file().with.contents.that.match(/^WHEEL_PREV_INDEX=5$/m);
});
it("should have WHEEL_CURRENT_INDEX , WHEEL_PREV_INDEX, WHEEL_NEXT_INDEX, WHEEL_FOR_START, WHEEL_FOR_END, and WHEEL_FOR_STEP in task under inner while component", ()=>{
expect(path.join(projectRootDir, "PS0_KEYWORD1_2","while0_1", "task0", logfilename)).to.be.a.file().with.contents.that.match(/^WHEEL_FOR_START=$/m);
expect(path.join(projectRootDir, "PS0_KEYWORD1_2","while0_1", "task0", logfilename)).to.be.a.file().with.contents.that.match(/^WHEEL_FOR_END=$/m);
expect(path.join(projectRootDir, "PS0_KEYWORD1_2","while0_1", "task0", logfilename)).to.be.a.file().with.contents.that.match(/^WHEEL_FOR_STEP=$/m);
expect(path.join(projectRootDir, "PS0_KEYWORD1_2","while0_1", "task0", logfilename)).to.be.a.file().with.contents.that.match(/^WHEEL_CURRENT_INDEX=1$/m);
expect(path.join(projectRootDir, "PS0_KEYWORD1_2","while0_1", "task0", logfilename)).to.be.a.file().with.contents.that.match(/^WHEEL_NEXT_INDEX=2$/m);
expect(path.join(projectRootDir, "PS0_KEYWORD1_2","while0_1", "task0", logfilename)).to.be.a.file().with.contents.that.match(/^WHEEL_PREV_INDEX=0$/m);
});
it("should have WHEEL_CURRENT_INDEX , WHEEL_PREV_INDEX, WHEEL_NEXT_INDEX, WHEEL_FOR_START, WHEEL_FOR_END, and WHEEL_FOR_STEP in task under inner foreach component", ()=>{
expect(path.join(projectRootDir, "PS0_KEYWORD1_2","foreach0_bar", "task0", logfilename)).to.be.a.file().with.contents.that.match(/^WHEEL_FOR_START=$/m);
expect(path.join(projectRootDir, "PS0_KEYWORD1_2","foreach0_bar", "task0", logfilename)).to.be.a.file().with.contents.that.match(/^WHEEL_FOR_END=$/m);
expect(path.join(projectRootDir, "PS0_KEYWORD1_2","foreach0_bar", "task0", logfilename)).to.be.a.file().with.contents.that.match(/^WHEEL_FOR_STEP=$/m);
expect(path.join(projectRootDir, "PS0_KEYWORD1_2","foreach0_bar", "task0", logfilename)).to.be.a.file().with.contents.that.match(/^WHEEL_CURRENT_INDEX=bar$/m);
expect(path.join(projectRootDir, "PS0_KEYWORD1_2","foreach0_bar", "task0", logfilename)).to.be.a.file().with.contents.that.match(/^WHEEL_NEXT_INDEX=$/m);
expect(path.join(projectRootDir, "PS0_KEYWORD1_2","foreach0_bar", "task0", logfilename)).to.be.a.file().with.contents.that.match(/^WHEEL_PREV_INDEX=foo$/m);
});
});

0 comments on commit 86f8a64

Please sign in to comment.