Skip to content

Commit

Permalink
✨E2E: add playwright sleepers test in CI (#5177)
Browse files Browse the repository at this point in the history
  • Loading branch information
sanderegg authored Jan 9, 2024
1 parent 87a98f1 commit 52e3fda
Show file tree
Hide file tree
Showing 11 changed files with 417 additions and 13 deletions.
67 changes: 67 additions & 0 deletions .github/workflows/ci-testing-deploy.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2184,6 +2184,72 @@ jobs:
if: always()
run: ./ci/github/system-testing/e2e.bash clean_up

system-test-e2e-playwright:
needs: [changes, build-test-images]
if: ${{ needs.changes.outputs.anything == 'true' || github.event_name == 'push' }}
timeout-minutes: 30 # if this timeout gets too small, then split the tests
name: "[sys] e2e-playwright"
runs-on: ${{ matrix.os }}
# NOTE: this is an interesting way, but generate a load of issues like not having docker installed, etc etc.
# container:
# image: mcr.microsoft.com/playwright/python:v1.39.0-jammy
strategy:
matrix:
python: ["3.10"]
os: [ubuntu-22.04]
docker_buildx: [v0.10.4]
fail-fast: false
steps:
- uses: actions/checkout@v4
- name: setup docker buildx
id: buildx
uses: docker/setup-buildx-action@v3
with:
version: ${{ matrix.docker_buildx }}
driver: docker-container
- name: setup python environment
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python }}
cache: "pip"
cache-dependency-path: "tests/e2e-playwright/requirements/dev.txt"
- name: expose github runtime for buildx
uses: crazy-max/ghaction-github-runtime@v3
- name: download docker images
uses: actions/download-artifact@v4
with:
name: docker-buildx-images-${{ runner.os }}-${{ github.sha }}
path: /${{ runner.temp }}/build
- name: load docker images
run: make load-images local-src=/${{ runner.temp }}/build
- name: prepare devenv
run: make devenv
- name: show system version
run: ./ci/helpers/show_system_versions.bash
- name: setup
run: |
./ci/github/system-testing/e2e-playwright.bash install
- name: test
run: |
./ci/github/system-testing/e2e-playwright.bash test
- name: dump docker logs
id: docker_logs_dump
if: failure()
run: ./ci/github/system-testing/e2e-playwright.bash dump_docker_logs
- name: upload docker logs
if: failure()
uses: actions/upload-artifact@v4
with:
name: ${{ github.job }}_docker_logs
path: ./tests/e2e-playwright/test_failures
- name: upload tracing if failed
if: always()
uses: actions/upload-artifact@v4
with:
name: ${{ github.job }}_tracing
path: tests/e2e-playwright/test-results


system-test-environment-setup:
timeout-minutes: 30 # if this timeout gets too small, then split the tests
name: "[sys] environment setup"
Expand Down Expand Up @@ -2224,6 +2290,7 @@ jobs:
needs:
[
system-test-e2e,
system-test-e2e-playwright,
system-test-environment-setup,
system-test-public-api,
system-test-swarm-deploy,
Expand Down
49 changes: 49 additions & 0 deletions ci/github/system-testing/e2e-playwright.bash
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
#!/bin/bash
# http://redsymbol.net/articles/unofficial-bash-strict-mode/
# https://github.com/GoogleChrome/puppeteer/blob/master/docs/troubleshooting.md#running-puppeteer-on-travis-ci
set -o errexit # abort on nonzero exitstatus
set -o nounset # abort on unbound variable
set -o pipefail # don't hide errors within pipes
IFS=$'\n\t'

# in case it's a Pull request, the env are never available, default to itisfoundation to get a maybe not too old version for caching
DOCKER_IMAGE_TAG=$(exec ci/helpers/build_docker_image_tag.bash)
export DOCKER_IMAGE_TAG

install() {
make devenv
source .venv/bin/activate
pushd tests/e2e-playwright
make install-ci-up-simcore
popd
}

test() {
source .venv/bin/activate
pushd tests/e2e-playwright
make test-sleepers
popd
}

dump_docker_logs() {
# get docker logs
# NOTE: Timeout avoids issue with dumping logs that hang!
out_dir=tests/e2e-playwright/test_failures
mkdir --parents "$out_dir"

for service_id in $(docker service ls -q); do
service_name=$(docker service inspect "$service_id" --format="{{.Spec.Name}}")
echo "Dumping logs for $service_name"
(timeout 30 docker service logs --timestamps --tail=400 --details "$service_id" >"$out_dir/$service_name.log" 2>&1) || true
done
}

# Check if the function exists (bash specific)
if declare -f "$1" >/dev/null; then
# call arguments verbatim
"$@"
else
# Show a helpful error
echo "'$1' is not a known function name" >&2
exit 1
fi
Original file line number Diff line number Diff line change
Expand Up @@ -174,6 +174,7 @@ qx.Class.define("osparc.workbench.WorkbenchUI", {

__addDesktop: function() {
const desktop = this.__desktop = new qx.ui.window.Desktop(new qx.ui.window.Manager());
osparc.utils.Utils.setIdToWidget(desktop, "desktopWindow");
this.__workbenchLayout.add(desktop, {
left: 0,
top: 0,
Expand Down
46 changes: 44 additions & 2 deletions tests/e2e-playwright/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,40 @@ leave build:
$(MAKE_C) $(REPO_BASE_DIR) $@


SIMCORE_DOT_ENV = $(abspath $(CURDIR)/../../.env)

define _up_simcore
# set some parameters to allow for e2e to run
echo LOGIN_REGISTRATION_INVITATION_REQUIRED=0 >> $(SIMCORE_DOT_ENV)
echo LOGIN_REGISTRATION_CONFIRMATION_REQUIRED=0 >> $(SIMCORE_DOT_ENV)
echo DEFAULT_MAX_NANO_CPUS=1000000000 >> $(SIMCORE_DOT_ENV)
echo DEFAULT_MAX_MEMORY=134217728 >> $(SIMCORE_DOT_ENV)
echo SIDECAR_FORCE_CPU_NODE=1 >> $(SIMCORE_DOT_ENV)
$(MAKE_C) $(REPO_BASE_DIR) up-prod ops_ci=1
endef

SERVICE_IMAGE_NAME = "sleeper"
SERVICE_VERSION = "2.1.6"
SERVICE_IMAGE_TAG = $(SERVICE_IMAGE_NAME):$(SERVICE_VERSION)

define _transfer-images-to-registry
# pushing sleeper image
@docker pull itisfoundation/$(SERVICE_IMAGE_TAG)
@docker tag itisfoundation/$(SERVICE_IMAGE_TAG) registry:5000/simcore/services/comp/itis/$(SERVICE_IMAGE_TAG)
@docker push registry:5000/simcore/services/comp/itis/$(SERVICE_IMAGE_TAG)
# completed transfer of images
@curl -s registry:5000/v2/_catalog | jq '.repositories'
@curl -s http://registry:5000/v2/simcore/services/comp/itis/$(SERVICE_IMAGE_NAME)/tags/list?n=50 | jq '.'
endef

define _give_service_access_rights
@docker exec \
$$(docker ps -q --filter="name=postgres") \
psql --user scu --dbname simcoredb --command \
"INSERT INTO services_access_rights (key, version, gid, execute_access, write_access, product_name) \
VALUES ('simcore/services/comp/itis/$(SERVICE_IMAGE_NAME)', '$(SERVICE_VERSION)', 1, TRUE, FALSE, 'osparc');"
endef

# LOCAL ------------------

.PHONY: requirements
Expand All @@ -26,6 +60,13 @@ install-dev install-prod install-ci: _check_venv_active ## install app in develo
# installing playwright dependencies
@playwright install

install-ci-up-simcore: install-ci
@$(MAKE_C) $(REPO_BASE_DIR) local-registry
@$(_up_simcore)
@$(VENV_DIR)/bin/python utils/wait_for_services.py
@$(_transfer-images-to-registry)
@$(_give_service_access_rights)


get_my_ip := $(shell hostname --all-ip-addresses | cut --delimiter=" " --fields=1)

Expand All @@ -38,7 +79,8 @@ test-sleepers: _check_venv_active ## runs sleepers test on local deploy
--pdb \
--product-url=http://$(get_my_ip):9081 \
--autoregister \
$(CURDIR)/tests/sleepers.py
--tracing=retain-on-failure \
$(CURDIR)/tests/sleepers/sleepers.py


.PHONY: test-sleepers-dev
Expand All @@ -50,7 +92,7 @@ test-sleepers-dev: _check_venv_active ## runs sleepers test on local deploy
--product-url=http://$(get_my_ip):9081 \
--headed \
--autoregister \
$(CURDIR)/tests/sleepers.py
$(CURDIR)/tests/sleepers/sleepers.py



Expand Down
2 changes: 2 additions & 0 deletions tests/e2e-playwright/requirements/_test.in
Original file line number Diff line number Diff line change
@@ -1,8 +1,10 @@
docker
faker
pydantic[email]
pytest-html
pytest-instafail
pytest-playwright
pytest-runner
pytest-sugar
pyyaml
tenacity
13 changes: 11 additions & 2 deletions tests/e2e-playwright/requirements/_test.txt
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@ charset-normalizer==3.3.2
# via requests
dnspython==2.4.2
# via email-validator
docker==7.0.0
# via -r requirements/_test.in
email-validator==2.1.0.post1
# via pydantic
exceptiongroup==1.2.0
Expand All @@ -32,6 +34,7 @@ markupsafe==2.1.3
# via jinja2
packaging==23.2
# via
# docker
# pytest
# pytest-sugar
playwright==1.40.0
Expand Down Expand Up @@ -70,8 +73,12 @@ python-dateutil==2.8.2
# via faker
python-slugify==8.0.1
# via pytest-playwright
pyyaml==6.0.1
# via -r requirements/_test.in
requests==2.31.0
# via pytest-base-url
# via
# docker
# pytest-base-url
six==1.16.0
# via python-dateutil
tenacity==8.2.3
Expand All @@ -88,4 +95,6 @@ typing-extensions==4.9.0
# pydantic-core
# pyee
urllib3==2.1.0
# via requests
# via
# docker
# requests
1 change: 1 addition & 0 deletions tests/e2e-playwright/requirements/_tools.txt
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,7 @@ pyproject-hooks==1.0.0
pyyaml==6.0.1
# via
# -c requirements/../../../requirements/constraints.txt
# -c requirements/_test.txt
# pre-commit
ruff==0.1.8
# via -r requirements/../../../requirements/devenv.txt
Expand Down
2 changes: 1 addition & 1 deletion tests/e2e-playwright/tests/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -149,7 +149,7 @@ def register(
product_url: AnyUrl,
user_name: str,
user_password: str,
) -> Iterator[Callable[[], AutoRegisteredUser]]:
) -> Callable[[], AutoRegisteredUser]:
def _do() -> AutoRegisteredUser:
print(
f"------> Registering in {product_url=} using {user_name=}/{user_password=}"
Expand Down
35 changes: 35 additions & 0 deletions tests/e2e-playwright/tests/sleepers/conftest.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
import pytest


def pytest_addoption(parser: pytest.Parser) -> None:
group = parser.getgroup(
"oSparc e2e options", description="oSPARC-e2e specific parameters"
)
group.addoption(
"--num-sleepers",
action="store",
type=int,
default=2,
help="URL pointing to the deployment to be tested",
)

group.addoption(
"--input-sleep-time",
action="store",
type=int,
default=None,
help="URL pointing to the deployment to be tested",
)


@pytest.fixture
def num_sleepers(request: pytest.FixtureRequest) -> int:
num = request.config.getoption("--num-sleepers")
assert isinstance(num, int)
return num


@pytest.fixture
def input_sleep_time(request: pytest.FixtureRequest) -> int | None:
sleep_time = request.config.getoption("--input-sleep-time")
return sleep_time
Original file line number Diff line number Diff line change
Expand Up @@ -21,10 +21,9 @@
)
from tenacity import retry, retry_if_exception_type, stop_after_delay, wait_fixed

_NUM_SLEEPERS = 12
_WAITING_FOR_CLUSTER_MAX_WAITING_TIME: Final[int] = 5 * MINUTE
_WAITING_FOR_STARTED_MAX_WAITING_TIME: Final[int] = 5 * MINUTE
_WAITING_FOR_SUCCESS_MAX_WAITING_TIME: Final[int] = _NUM_SLEEPERS * 1 * MINUTE
_WAITING_FOR_SUCCESS_MAX_WAITING_TIME_PER_SLEEPER: Final[int] = 1 * MINUTE
_WAITING_FOR_FILE_NAMES_MAX_WAITING_TIME: Final[
datetime.timedelta
] = datetime.timedelta(seconds=30)
Expand All @@ -51,12 +50,14 @@ def _get_file_names(page: Page) -> list[str]:
return file_names_found


def test_sleepers(
def test_sleepers( # noqa: PLR0915
page: Page,
log_in_and_out: WebSocket,
create_new_project_and_delete: Callable[..., None],
start_and_stop_pipeline: Callable[..., SocketIOEvent],
product_billable: bool,
num_sleepers: int,
input_sleep_time: int | None,
):
# open service tab and filter for sleeper
page.get_by_test_id("servicesTabBtn").click()
Expand All @@ -70,10 +71,25 @@ def test_sleepers(
create_new_project_and_delete(auto_delete=True)

# we are now in the workbench
for _ in range(1, _NUM_SLEEPERS):
for _ in range(1, num_sleepers):
page.get_by_text("New Node").click()
page.get_by_text("sleeperA service which").click()
page.get_by_text("Add", exact=True).click()
page.get_by_placeholder("Filter").click()
page.get_by_placeholder("Filter").fill("sleeper")
page.get_by_placeholder("Filter").press("Enter")

# set inputs if needed
if input_sleep_time:
for index, sleeper in enumerate(page.get_by_test_id("nodeTreeItem").all()[1:]):
print(f"---> setting sleeper {index} input time to {input_sleep_time}...")
sleeper.click()
sleep_interval_selector = page.get_by_role("textbox").nth(1)
sleep_interval_selector.click()
sleep_interval_selector.fill(f"{input_sleep_time}")
print(f"<--- sleeper {index} input time set to {input_sleep_time}")

workbench_selector = page.get_by_test_id("desktopWindow")
assert workbench_selector
workbench_selector.click()

# start the pipeline (depending on the state of the cluster, we might receive one of
# in [] are optional states depending on the state of the clusters and if we have external clusters
Expand Down Expand Up @@ -117,7 +133,9 @@ def test_sleepers(
waiter = SocketIOProjectStateUpdatedWaiter(expected_states=("SUCCESS", "FAILED"))
print("---> waiting for SUCCESS state...")
with log_in_and_out.expect_event(
"framereceived", waiter, timeout=_WAITING_FOR_SUCCESS_MAX_WAITING_TIME
"framereceived",
waiter,
timeout=num_sleepers * _WAITING_FOR_SUCCESS_MAX_WAITING_TIME_PER_SLEEPER,
) as event:
...
current_state = decode_socketio_42_message(event.value).obj["data"]["state"][
Expand All @@ -129,7 +147,7 @@ def test_sleepers(
# check the outputs (the first item is the title, so we skip it)
expected_file_names = ["logs.zip", "single_number.txt"]
print(
f"---> looking for {expected_file_names=} in all {_NUM_SLEEPERS} sleeper services..."
f"---> looking for {expected_file_names=} in all {num_sleepers} sleeper services..."
)

for index, sleeper in enumerate(page.get_by_test_id("nodeTreeItem").all()[1:]):
Expand Down
Loading

0 comments on commit 52e3fda

Please sign in to comment.