From 9e97f7b0f688c302825a375d37e0d36eef9005e0 Mon Sep 17 00:00:00 2001 From: Kim Pevey Date: Tue, 24 Oct 2023 08:05:16 -0500 Subject: [PATCH] ENH - Add Playwright for automated testing (#293) * add playwright files, add github ci test * fix typo * fix another typo... * take a different approach to yarn start * take a different approach to yarn start * gh action syntax fix * add wait-for-it * ensure conda env is activated * yarm immutable installs false * screenshot is a forbidden pytest option, create .env file * switch to expect new ui on 8080 * switch to look on port 8081 for the test * i obviously didn't know how to envsubst works * why does this work differently on my machine??? * switch back to 5000 for conda-store-server and 8080 for the ui * add a long timeout for the first page load to ensure startup is complete * use 5000 for server port * one. more. try. * cleanup * env var in ci * gh actions doesnt support env vars in env vars * add a few more doc strings * attempt to fix mamba issues with unknown symbol * fix for setup-miniconda solver issues --- .env.tpl | 8 + .github/workflows/test.yml | 77 +++++++++ .gitignore | 3 + environment_dev.yml | 14 ++ test/playwright/conftest.py | 12 ++ test/playwright/test_ux.py | 325 ++++++++++++++++++++++++++++++++++++ 6 files changed, 439 insertions(+) create mode 100644 .env.tpl create mode 100644 .github/workflows/test.yml create mode 100755 environment_dev.yml create mode 100644 test/playwright/conftest.py create mode 100644 test/playwright/test_ux.py diff --git a/.env.tpl b/.env.tpl new file mode 100644 index 00000000..8b23ef41 --- /dev/null +++ b/.env.tpl @@ -0,0 +1,8 @@ +REACT_APP_API_URL=$REACT_APP_API_URL +REACT_APP_AUTH_METHOD=$REACT_APP_AUTH_METHOD +REACT_APP_LOGIN_PAGE_URL=$REACT_APP_LOGIN_PAGE_URL +REACT_APP_AUTH_TOKEN=$REACT_APP_AUTH_TOKEN +REACT_APP_STYLE_TYPE=$REACT_APP_STYLE_TYPE +REACT_APP_CONTEXT=$REACT_APP_CONTEXT +REACT_APP_SHOW_AUTH_BUTTON=$REACT_APP_SHOW_AUTH_BUTTON +REACT_APP_LOGOUT_PAGE_URL=$REACT_APP_LOGOUT_PAGE_URL diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 00000000..54708819 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,77 @@ +name: Playwright Tests + +env: + CONDA_STORE_SERVER_PORT: 8080 + CONDA_STORE_BASE_URL: http://localhost:8080 + CONDA_STORE_AUTH: basic + CONDA_STORE_USERNAME: username + CONDA_STORE_PASSWORD: password + REACT_APP_API_URL: http://localhost:5000/conda-store/ + REACT_APP_AUTH_METHOD: cookie + REACT_APP_LOGIN_PAGE_URL: http://localhost:5000/conda-store/login?next= + REACT_APP_AUTH_TOKEN: + REACT_APP_STYLE_TYPE: green-accent + REACT_APP_CONTEXT: webapp + REACT_APP_SHOW_AUTH_BUTTON: true + REACT_APP_LOGOUT_PAGE_URL: http://localhost:5000/conda-store/logout?next=/ + YARN_ENABLE_IMMUTABLE_INSTALLS: false + +on: + pull_request: + push: + branches: + - main + +jobs: + test-conda-store-ui: + name: 'unit-test conda-store-ui' + strategy: + matrix: + # cannot run on windows due to needing fake-chroot for conda-docker + # osx takes forever to get a scheduled job + os: [ubuntu-latest] + runs-on: ${{ matrix.os }} + defaults: + run: + shell: bash -el {0} + steps: + - name: 'Checkout Repository' + uses: actions/checkout@master + + - name: Set up Python + uses: conda-incubator/setup-miniconda@v2 + env: + CONDA_SOLVER: libmamba + with: + activate-environment: test-env + environment-file: environment_dev.yml + auto-activate-base: false + + - name: Install Dependencies + run: | + sudo apt install wait-for-it -y + + - name: Deploy conda-store-server docker container + run: | + docker-compose -f docker-compose-dev.yml up -d --build + docker ps + + wait-for-it localhost:5000 # conda-store-server + + + - name: Deploy webpack dev server + shell: bash -el {0} + run: | + conda list + playwright install chromium + yarn install + yarn run build + yarn run start & + pytest --video on --output test-results --screenshots true test/playwright/test_ux.py + + - name: Upload artifacts + uses: actions/upload-artifact@v2 + if: ${{ always() }} + with: + name: playwright-tests + path: test-results diff --git a/.gitignore b/.gitignore index 87a392b0..5c10c913 100644 --- a/.gitignore +++ b/.gitignore @@ -149,3 +149,6 @@ coverage *.swp src/version.ts + +# playwright screenshots +static diff --git a/environment_dev.yml b/environment_dev.yml new file mode 100755 index 00000000..bdbcdadc --- /dev/null +++ b/environment_dev.yml @@ -0,0 +1,14 @@ +# Environment for developing the conda-store-ui and for running +# playwright tests + +name: test-env +channels: +- conda-forge +dependencies: +- python=3.10 +- yarn +- nodejs==16.14.2 +- pytest +- pip: + - playwright + - pytest-playwright \ No newline at end of file diff --git a/test/playwright/conftest.py b/test/playwright/conftest.py new file mode 100644 index 00000000..1b09f67e --- /dev/null +++ b/test/playwright/conftest.py @@ -0,0 +1,12 @@ +import pytest + + +def pytest_addoption(parser): + parser.addoption("--screenshots", action="store", default="false") + +@pytest.fixture(scope="session") +def screenshot(pytestconfig): + if pytestconfig.getoption("screenshots") == 'false': + return False + else: + return True diff --git a/test/playwright/test_ux.py b/test/playwright/test_ux.py new file mode 100644 index 00000000..1b1f8e4c --- /dev/null +++ b/test/playwright/test_ux.py @@ -0,0 +1,325 @@ +"""Test suite for user interactions with the UI. It is designed to run both +inside and outside of pytest to make future development easier. +""" +import os +import requests +import time + +import pytest +from playwright.sync_api import Page +from playwright.sync_api import sync_playwright +import random + + +CONDA_STORE_SERVER_PORT = os.environ.get( + "CONDA_STORE_SERVER_PORT", f"8080" +) +CONDA_STORE_BASE_URL = os.environ.get( + "CONDA_STORE_BASE_URL", f"http://localhost:{CONDA_STORE_SERVER_PORT}" +) +CONDA_STORE_USERNAME = os.environ.get("CONDA_STORE_USERNAME", "username") +CONDA_STORE_PASSWORD = os.environ.get("CONDA_STORE_PASSWORD", "password") + + +@pytest.fixture +def test_config(): + return { + 'base_url': CONDA_STORE_BASE_URL, + 'username': CONDA_STORE_USERNAME, + 'password': CONDA_STORE_PASSWORD, + 'server_port': CONDA_STORE_SERVER_PORT, + } + + +def _login_sequence(page, screenshot=False): + """Conda-store ui login sequence. From the default UI interface, click log + in and go through the log in UI on the following page. The UI will be + returned back to the default UI. + + Parameters + ---------- + page: playwright.Page + page object for the current test being run + screenshot: bool + [Optional] Flag to trigger screenshot collection, set to True to + grab screenshots + """ + # Log in sequence + # Click Login + page.locator("text=Log in").click() + + if screenshot: + page.screenshot(path="test-results/conda-store-login_screen.png") + + # Fill in the Username field + page.locator('[placeholder="Username"]').fill("username") + + # Fill in the Password field + page.locator('[placeholder="Password"]').fill("password") + + with page.expect_navigation(): + page.locator('button:has-text("Sign In")').click() + + if screenshot: + page.screenshot(path="test-results/conda-store-authenticated.png") + + +def _create_new_environment(page, screenshot=False): + """Workflow to create a new environment in the UI. The env will be + in the "username" workspace and will have a semi-random number to + ensure that the env is indeed new since if the environment already + exists we get a different UI. This allows this test to be run multiple + times without needing to empty the database. + + Note: this environment takes about a minute to create + WARNING: Changes to this method will require reflective changes on + `_existing_environment_interactions` since it uses this env. + + Parameters + ---------- + page: playwright.Page + page object for the current test being run + screenshot: bool + [Optional] Flag to trigger screenshot collection, set to True to + grab screenshots + """ + # ensure new filename in case this test is run multiple times + new_env_name = f'test_env_{random.randint(0, 100000)}' + # set timeout for building the environment + time_to_build_env = 2 * 60 * 1000 # 2 minutes in milliseconds + + # Create the new environment + # click the + to create a new env + page.get_by_label("Create a new environment in the username namespace").click() + if screenshot: + page.screenshot(path="test-results/conda-store-new-env.png") + # fill in the env name + page.get_by_placeholder("Environment name").fill(new_env_name) + # fill in the description + page.get_by_placeholder("Enter here the description of your environment").fill("description") + # click the + to add a package + page.get_by_role("button", name="+ Add Package").click() + # add a package to the ui + page.get_by_label("Enter package").fill("rich") + page.get_by_role("option", name="rich", exact=True).click() + # open up the channels accordian card + page.get_by_role("button", name="Channels").click() + # click the + to add a channel + page.get_by_role("button", name="+ Add Channel").click() + # fill in conda-forge as the new channel name + page.get_by_label("Enter channel").fill("conda-forge") + # press enter to submit the channel to the list + page.get_by_label("Enter channel").press("Enter") + # click create to start building the env + page.get_by_role("button", name="Create", exact=True).click() + + # Interact with the environment shortly after creation + # click to open the Active environment dropdown manu + page.get_by_role("button", name=" - Active", exact=False).click() + # click on the Active environment on the dropdown menu item (which is currently building) + page.get_by_role("option", name=" - Active", exact=False).click() + # ensure that the environment is building + assert page.get_by_text("Building").is_visible() + # wait until the status is `Completed` + completed = page.get_by_text("Completed", exact=False) + completed.wait_for(state='attached', timeout=time_to_build_env) + assert completed.is_visible() + + return new_env_name + + +def _close_environment_tabs(page): + """Close any open tabs in the UI. This will continue closing tabs + until no tabs remain open. + + Paramaters + ---------- + page: playwright.Page + page object for the current test being run + """ + close_tab = page.get_by_test_id("closeTab") + while close_tab.count() > 0: + close_tab.first.click() + + +def _existing_environment_interactions(page, env_name, time_to_build_env=3*60*1000, screenshot=False): + """test interactions with existing environments. + During this test, the test will be rebuilt twice. + + Note: This test assumes the environment being tested is the one from + `_create_new_environment`. Changes to that method will require changes + here as well (expected existing packages, etc). + + Parameters + ---------- + page: playwright.Page + page object for the current test being run + env_name: str + Name of existing environment to interact with - must already exist! + time_to_build_env: float + [Optional] Time to wait for an updated environment to rebuild in ms + screenshot: bool + [Optional] Flag to trigger screenshot collection, set to True to + grab screenshots + + """ + # edit existing environment throught the YAML editor + page.get_by_role("button", name=env_name).click() + page.get_by_role("button", name="Edit").click() + page.get_by_label("Switch to YAML Editor").check() + if screenshot: + page.screenshot(path="test-results/conda-store-yaml-editor.png") + page.get_by_text("- rich").click() + page.get_by_text("channels: - conda-forgedependencies: - rich - pip: - nothing - ipykernel").fill("channels:\n - conda-forge\ndependencies:\n - rich\n - python\n - pip:\n - nothing\n - ipykernel\n\n") + page.get_by_role("button", name="Save").click() + # wait until the status is `Completed` + completed = page.get_by_text("Completed", exact=False) + completed.wait_for(state='attached', timeout=time_to_build_env) + + # ensure the namespace is expanded + if not page.get_by_role("button", name=env_name).is_visible(): + # click to expand the `username` name space (but not click the +) + page.get_by_role("button", name="username Create a new environment in the username namespace").click() + + # edit existing environment + page.get_by_role("button", name=env_name).click() + page.get_by_role("button", name="Edit").click() + # page.get_by_placeholder("Enter here the description of your environment").click() + # change the description + page.get_by_placeholder("Enter here the description of your environment").fill("new description") + # change the vesion spec of an existing package + page.get_by_role("row", name="ipykernel", exact=False).get_by_role("button").first.click() + page.get_by_role("option", name=">=").click() + # Note: purposefully not testing version constraint since there is inconsistent behavior here + + # add a new package + page.get_by_role("button", name="+ Add Package").click() + page.get_by_label("Enter package").fill("click") + page.get_by_role("option", name="click", exact=True).click() + # Note: purposefully not testing version constraint since there is inconsistent behavior here + + # delete a package + page.get_by_role("row", name="rich", exact=False).get_by_test_id("RemovePackageTest").click() + + # promote a package installed as dependency to specified package + page.locator("#infScroll > .infinite-scroll-component__outerdiv > .infinite-scroll-component > div > div > .MuiButtonBase-root").first.click() + + # delete conda-forge channel + page.get_by_test_id("DeleteIcon").click() + # add conda-forge channel + page.get_by_role("button", name="+ Add Channel").click() + page.get_by_label("Enter channel").fill("conda-forge") + page.get_by_label("Enter channel").press("Enter") + # click save to start the new env build + page.get_by_role("button", name="Save").click() + + # wait until the status is `Completed` + completed = page.get_by_text("Completed", exact=False) + completed.wait_for(state='attached', timeout=time_to_build_env) + + + # Edit -> Cancel editing + page.get_by_role("button", name=env_name).click() + page.get_by_role("button", name="Edit").click() + page.get_by_role("button", name="Cancel").click() + + # Edit -> Delete environment + page.get_by_role("button", name="Edit").click() + page.get_by_text("Delete environment").click() + page.get_by_role("button", name="Delete").click() + + assert not page.get_by_role("button", name=env_name).is_visible() + + + +def test_integration(page: Page, test_config, screenshot): + """Basic integration test. + + When this test runs in CI, we launch the webpack server as a detached + service at the same time that this test is run. For this reason, we + have a try/except here to allow the webpack server to finish deploying + before the test begins. + + Parameters + ---------- + page: playwright.Page + page object for the current test being run + test_config: + Fixture containing the configuration env vars + screenshot: bool + Fixture flag to trigger screenshot collection, set to True to + grab screenshots + """ + # wait for server to spin up if necessary + server_running = False + retry_wait_time = 2 # seconds + max_wait_time = 4 * 60 # 4 minutes + elapsed_wait_time = 0 + # loop until server is running or max_wait_time is reached + while not server_running and elapsed_wait_time < max_wait_time: + try: + requests.head(test_config['base_url'], allow_redirects=True).status_code != 200 + server_running = True + except requests.exceptions.ConnectionError: + elapsed_wait_time += retry_wait_time + time.sleep(retry_wait_time) + + # Go to http://localhost:{server_port} + page.goto(test_config['base_url'], wait_until="domcontentloaded", timeout=4*60*1000) + + page.screenshot(path="test-results/conda-store-unauthenticated.png") + if screenshot: + page.screenshot(path="test-results/conda-store-unauthenticated.png") + + # Log in to conda-store + _login_sequence(page, screenshot=screenshot) + + # create a new environment + env_name = _create_new_environment(page, screenshot=screenshot) + + # close any open tabs on the conda-store ui + _close_environment_tabs(page) + + # interact with an existing environment + _existing_environment_interactions(page, env_name, screenshot=screenshot) + + +if __name__ == "__main__": + """This sequence runs through the basic UI test outside of pytest to allow + for more control during development. It is not intended to be used in CI. + """ + + config = { + 'base_url': f"http://localhost:{CONDA_STORE_SERVER_PORT}", + 'username': CONDA_STORE_USERNAME, + 'password': CONDA_STORE_PASSWORD, + 'server_port': CONDA_STORE_SERVER_PORT, + } + screenshot = False + + # ######################################################################## + # Start playwright and setup + playwright = sync_playwright().start() + # Use playwright.chromium, playwright.firefox or playwright.webkit + # Pass headless=False to launch() to see the browser UI + # slow_mo adds milliseconds to each playwright command so humans can follow along + browser = playwright.chromium.launch(headless=False, slow_mo=500) + page = browser.new_page() + + # Go to http://localhost:{server_port} + page.goto(config['base_url'], wait_until="domcontentloaded") + + # Log in to conda-store + _login_sequence(page) + + # create a new environment + env_name = _create_new_environment(page, screenshot=screenshot) + + # close any open tabs on the conda-store ui + _close_environment_tabs(page) + + # interact with an existing environment + _existing_environment_interactions(page, env_name, screenshot=screenshot) + + browser.close() + playwright.stop() \ No newline at end of file