diff --git a/.github/workflows/smoke_tests.yml b/.github/workflows/smoke_tests.yml index 7342d38d..bd4450f5 100644 --- a/.github/workflows/smoke_tests.yml +++ b/.github/workflows/smoke_tests.yml @@ -7,6 +7,7 @@ on: jobs: build: + name: Smoke tests runs-on: ubuntu-latest steps: @@ -19,8 +20,10 @@ jobs: python-version: "3.10" - name: Install dependencies - run: | - pipx run --spec poetry==1.6.1 poetry install - echo dupa - env: - POETRY_VIRTUALENVS_CREATE: false + run: pipx run --spec poetry==1.6.1 poetry install --no-ansi + + - name: Call `ray up` + run: pipx run poetry run ray up golem-cluster-dev.yaml -y + + - name: Call `ray down` + run: ray down golem-cluster-dev.yaml -y diff --git a/ray_on_golem/server/services/yagna.py b/ray_on_golem/server/services/yagna.py index 1156ac1e..c42c56f8 100644 --- a/ray_on_golem/server/services/yagna.py +++ b/ray_on_golem/server/services/yagna.py @@ -148,7 +148,13 @@ async def _stop_yagna_service(self) -> None: logger.info("Stopping Yagna done") async def _run_yagna_payment_fund(self) -> None: - await run_subprocess_output(self._yagna_path, "payment", "fund") + for _ in range(5): + try: + await run_subprocess_output(self._yagna_path, "payment", "fund") + except RayOnGolemError as e: + logger.error('failed: %s', e) + else: + return async def _get_or_create_yagna_appkey(self) -> str: if YAGNA_APPKEY: diff --git a/ray_on_golem/server/settings.py b/ray_on_golem/server/settings.py index a7735311..dac831f1 100644 --- a/ray_on_golem/server/settings.py +++ b/ray_on_golem/server/settings.py @@ -96,7 +96,7 @@ YAGNA_START_DEADLINE = timedelta(minutes=2) YAGNA_CHECK_DEADLINE = timedelta(seconds=2) -RAY_ON_GOLEM_START_DEADLINE = timedelta(minutes=2, seconds=30) +RAY_ON_GOLEM_START_DEADLINE = timedelta(minutes=5) RAY_ON_GOLEM_CHECK_DEADLINE = timedelta(seconds=2) RAY_ON_GOLEM_SHUTDOWN_DELAY = timedelta(seconds=10) RAY_ON_GOLEM_SHUTDOWN_DEADLINE = timedelta(seconds=30) diff --git a/ray_on_golem/utils.py b/ray_on_golem/utils.py index 21adb942..cef87746 100644 --- a/ray_on_golem/utils.py +++ b/ray_on_golem/utils.py @@ -36,7 +36,7 @@ async def run_subprocess_output(*args) -> bytes: stdout, stderr = await process.communicate() if process.returncode != 0: - raise RayOnGolemError(stderr) + raise RayOnGolemError(f'Process exited with code `{process.returncode}`!\nstdout:\n{stdout}\nstderr:\n{stderr}') return stdout