Skip to content

Commit

Permalink
Add scheduler unit test for propagation of dispatch info to jobs file
Browse files Browse the repository at this point in the history
  • Loading branch information
jonathan-eq committed Dec 14, 2023
1 parent 07818cf commit 60c08c2
Showing 1 changed file with 57 additions and 14 deletions.
71 changes: 57 additions & 14 deletions tests/unit_tests/scheduler/test_scheduler.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ def create_bash_step(script: str) -> ForwardModel:
)


def create_jobs_json(path: Path, steps: Sequence[ForwardModel]) -> None:
def create_jobs_json(realization: Realization) -> None:
jobs = {
"global_environment": {},
"config_path": "/dev/null",
Expand All @@ -33,39 +33,44 @@ def create_jobs_json(path: Path, steps: Sequence[ForwardModel]) -> None:
"executable": step.executable,
"argList": step.arglist,
}
for step in steps
for step in realization.forward_models
],
"run_id": "0",
"ert_pid": "0",
"real_id": "0",
"real_id": str(realization.iens),
}

with open(path / "jobs.json", "w") as f:
realization_run_path = Path(realization.run_arg.runpath)
realization_run_path.mkdir()
with open(realization_run_path / "jobs.json", "w") as f:
json.dump(jobs, f)


@pytest.fixture
def realization(storage, tmp_path):
ensemble = storage.create_experiment().create_ensemble(name="foo", ensemble_size=1)
return create_stub_realization(ensemble, tmp_path, 0)


def create_stub_realization(ensemble, base_path: Path, iens):
run_arg = RunArg(
run_id="",
ensemble_storage=ensemble,
iens=0,
iens=iens,
itr=0,
runpath=str(tmp_path),
runpath=str(base_path / str(iens)),
job_name="",
)

return Realization(
iens=0,
realization = Realization(
iens=iens,
forward_models=[],
active=True,
max_runtime=None,
run_arg=run_arg,
num_cpu=1,
job_script=str(shutil.which("job_dispatch.py")),
)
return realization


async def test_empty():
Expand All @@ -80,11 +85,13 @@ async def test_single_job(tmp_path: Path, realization):
sch = scheduler.Scheduler()
sch.add_realization(realization, callback_timeout=lambda _: None)

create_jobs_json(tmp_path, [step])
create_jobs_json(realization)
sch.add_dispatch_information_to_jobs_file()

assert await sch.execute() == EVTYPE_ENSEMBLE_STOPPED
assert (tmp_path / "testfile").read_text() == "Hello, world!\n"
assert (
tmp_path / str(realization.iens) / "testfile"
).read_text() == "Hello, world!\n"


async def test_cancel(tmp_path: Path, realization):
Expand All @@ -94,7 +101,7 @@ async def test_cancel(tmp_path: Path, realization):
sch = scheduler.Scheduler()
sch.add_realization(realization, callback_timeout=lambda _: None)

create_jobs_json(tmp_path, [step])
create_jobs_json(realization)
sch.add_dispatch_information_to_jobs_file()

scheduler_task = asyncio.create_task(sch.execute())
Expand All @@ -106,5 +113,41 @@ async def test_cancel(tmp_path: Path, realization):
sch.kill_all_jobs()
await scheduler_task

assert (tmp_path / "a").exists()
assert not (tmp_path / "b").exists()
assert (tmp_path / str(realization.iens) / "a").exists()
assert not (tmp_path / str(realization.iens) / "b").exists()


def test_add_dispatch_information_to_jobs_file(storage, tmp_path: Path):
test_ee_uri = "ws://test_ee_uri.com/121/"
test_ens_id = "test_ens_id121"
test_ee_token = "test_ee_token_t0k€n121"
test_ee_cert = "test_ee_cert121.pem"

ensemble_size = 10

ensemble = storage.create_experiment().create_ensemble(
name="foo", ensemble_size=ensemble_size
)
realizations = [
create_stub_realization(ensemble, tmp_path, iens)
for iens in range(ensemble_size)
]

sch = scheduler.Scheduler()
sch.set_ee_info(test_ee_uri, test_ens_id, test_ee_cert, test_ee_token)

for realization in realizations:
sch.add_realization(realization, callback_timeout=lambda _: None)
create_jobs_json(realization)

sch.add_dispatch_information_to_jobs_file()

for realization in realizations:
job_file_path = Path(realization.run_arg.runpath, "jobs.json")
content: dict = json.loads(job_file_path.read_text(encoding="utf-8"))
assert content["ens_id"] == test_ens_id
assert content["real_id"] == str(realization.iens)
assert content["dispatch_url"] == test_ee_uri
assert content["ee_token"] == test_ee_token
assert content["ee_cert_path"] == test_ee_cert
assert type(content["jobList"]) == list and len(content["jobList"]) == 0

0 comments on commit 60c08c2

Please sign in to comment.