Skip to content

Commit

Permalink
Add check pods are not scheduled when testing gang-scheduler integrat…
Browse files Browse the repository at this point in the history
…ions in e2e (#1835)

Signed-off-by: Yuki Iwai <[email protected]>
  • Loading branch information
tenzen-y authored Jun 18, 2023
1 parent 15ea54b commit e002b8a
Showing 1 changed file with 14 additions and 8 deletions.
22 changes: 14 additions & 8 deletions sdk/python/test/e2e/utils.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import logging
import time

from kubeflow.training import TrainingClient
from kubeflow.training.constants import constants
Expand All @@ -17,13 +18,19 @@ def verify_unschedulable_job_e2e(
logging.info(f"\n\n\n{job_kind} is creating")
client.wait_for_job_conditions(name, namespace, job_kind, {constants.JOB_CONDITION_CREATED})

# Job should have a Created condition.
if not client.is_job_created(name, namespace, job_kind):
raise Exception(f"{job_kind} should be in Created condition")
logging.info("Checking 3 times that pods are not scheduled")
for num in range(3):
logging.info(f"Number of attempts: {int(num)+1}/3")
# Job should have a Created condition.
if not client.is_job_created(name, namespace, job_kind):
raise Exception(f"{job_kind} should be in Created condition")

# Job shouldn't have a Running condition.
if client.is_job_running(name, namespace, job_kind):
raise Exception(f"{job_kind} shouldn't be in Running condition")
# Job shouldn't have a Running condition.
if client.is_job_running(name, namespace, job_kind):
raise Exception(f"{job_kind} shouldn't be in Running condition")

logging.info("Sleeping 5 seconds...")
time.sleep(5)


def verify_job_e2e(
Expand Down Expand Up @@ -68,8 +75,7 @@ def verify_job_e2e(
def get_pod_spec_scheduler_name(gang_scheduler_name: str) -> str:
if gang_scheduler_name == TEST_GANG_SCHEDULER_NAME_SCHEDULER_PLUGINS:
return DEFAULT_SCHEDULER_PLUGINS_NAME
# TODO (tenzen-y): Implement E2E tests using volcano.
elif gang_scheduler_name == TEST_GANG_SCHEDULER_NAME_VOLCANO:
return ""
return TEST_GANG_SCHEDULER_NAME_VOLCANO

return ""

0 comments on commit e002b8a

Please sign in to comment.