From 7276e12c80035593e982e20db9239d10c014d5cb Mon Sep 17 00:00:00 2001 From: Sawjan Gurung Date: Mon, 7 Oct 2024 12:19:09 +0545 Subject: [PATCH] [gui-tests][full-ci] retry failed scenario (#11920) * test: retry failed scenario * test: report helper * tests: use snake_case --- .drone.star | 8 +- test/gui/.pylintrc | 1 + test/gui/config.sample.ini | 2 +- test/gui/shared/scripts/bdd_hooks.py | 75 +++++-------------- .../shared/scripts/helpers/ConfigHelper.py | 31 ++++---- .../shared/scripts/helpers/ReportHelper.py | 75 +++++++++++++++++++ .../scripts/helpers/SetupClientHelper.py | 4 +- 7 files changed, 122 insertions(+), 74 deletions(-) create mode 100644 test/gui/shared/scripts/helpers/ReportHelper.py diff --git a/.drone.star b/.drone.star index add6235b5ee..be738ad8e9b 100644 --- a/.drone.star +++ b/.drone.star @@ -209,7 +209,11 @@ def gui_test_pipeline(ctx): "--tags ~@skipOnLinux", ] - if not "full-ci" in ctx.build.title.lower() and ctx.build.event == "pull_request": + # '--retry' and '--abortOnFail' are mutually exclusive + if "full-ci" in ctx.build.title.lower() or ctx.build.event in ("tag", "cron"): + # retry failed tests once + squish_parameters.append("--retry 1") + elif not "full-ci" in ctx.build.title.lower() and ctx.build.event == "pull_request": squish_parameters.append("--abortOnFail") if params.get("skip", False): @@ -334,7 +338,7 @@ def gui_tests(squish_parameters = "", server_type = "oc10"): "STACKTRACE_FILE": "%s/stacktrace.log" % dir["guiTestReport"], "PLAYWRIGHT_BROWSERS_PATH": "%s/.playwright" % dir["base"], "OWNCLOUD_CORE_DUMP": 1, - "SCREEN_RECORD_ON_FAILURE": False, + "RECORD_VIDEO_ON_FAILURE": False, # allow to use any available pnpm version "COREPACK_ENABLE_STRICT": 0, }, diff --git a/test/gui/.pylintrc b/test/gui/.pylintrc index 146f973f5ed..11c27e4440c 100644 --- a/test/gui/.pylintrc +++ b/test/gui/.pylintrc @@ -18,6 +18,7 @@ ignore-paths=^tst_.*/test.py$, shared/scripts/custom_lib ignored-modules= squish, + squishinfo, object, objectmaphelper, test, diff --git a/test/gui/config.sample.ini b/test/gui/config.sample.ini index 1c374b19dc1..23dc09f61f7 100644 --- a/test/gui/config.sample.ini +++ b/test/gui/config.sample.ini @@ -11,4 +11,4 @@ TEMP_FOLDER_PATH= CLIENT_CONFIG_DIR= GUI_TEST_REPORT_DIR= OCIS=false -SCREEN_RECORD_ON_FAILURE=false \ No newline at end of file +RECORD_VIDEO_ON_FAILURE=false \ No newline at end of file diff --git a/test/gui/shared/scripts/bdd_hooks.py b/test/gui/shared/scripts/bdd_hooks.py index a464c93ba18..588e03b2600 100644 --- a/test/gui/shared/scripts/bdd_hooks.py +++ b/test/gui/shared/scripts/bdd_hooks.py @@ -17,7 +17,6 @@ # manual for a complete reference of the available API. import shutil import os -import glob from urllib import request, error from datetime import datetime @@ -36,8 +35,9 @@ ) from helpers.api.utils import url_join from helpers.FilesHelper import prefix_path_namespace, cleanup_created_paths -from pageObjects.Toolbar import Toolbar +from helpers.ReportHelper import save_video_recording, take_screenshot +from pageObjects.Toolbar import Toolbar from pageObjects.AccountSetting import AccountSetting from pageObjects.AccountConnectionWizard import AccountConnectionWizard @@ -50,6 +50,7 @@ # this will reset in every test suite PREVIOUS_FAIL_RESULT_COUNT = 0 PREVIOUS_ERROR_RESULT_COUNT = 0 +PREVIOUS_SCENARIO = "" # runs before a feature @@ -65,6 +66,11 @@ def hook(context): def hook(context): unlock_keyring() clear_scenario_config() + global PREVIOUS_SCENARIO + if PREVIOUS_SCENARIO == context.title: + test.log("[INFO] Retrying this failed scenario...") + set_config("retrying", True) + PREVIOUS_SCENARIO = context.title # runs before every scenario @@ -142,44 +148,9 @@ def scenario_failed(): ) -def get_screenshot_name(title): - return title.replace(" ", "_").replace("/", "_").strip(".") + ".png" - - -def get_screenrecord_name(title): - return title.replace(" ", "_").replace("/", "_").strip(".") + ".mp4" - - -def save_screenrecord(filename): - try: - # do not throw if stopVideoCapture() fails - test.stopVideoCapture() - except: - test.log("Failed to stop screen recording") - - if not (video_dir := squishinfo.resultDir): - video_dir = squishinfo.testCase - else: - test_case = "/".join(squishinfo.testCase.split("/")[-2:]) - video_dir = os.path.join(video_dir, test_case) - video_dir = os.path.join(video_dir, "attachments") - - if scenario_failed(): - video_files = glob.glob(f"{video_dir}/**/*.mp4", recursive=True) - screenrecords_dir = os.path.join( - get_config("guiTestReportDir"), "screenrecords" - ) - if not os.path.exists(screenrecords_dir): - os.makedirs(screenrecords_dir) - # reverse the list to get the latest video first - video_files.reverse() - for idx, video in enumerate(video_files): - if idx: - file_parts = filename.rsplit(".", 1) - filename = f"{file_parts[0]}_{idx+1}.{file_parts[1]}" - shutil.move(video, os.path.join(screenrecords_dir, filename)) - - shutil.rmtree(prefix_path_namespace(video_dir)) +def scenario_title_to_filename(title): + # scenario name can have "/" which is invalid filename + return title.replace(" ", "_").replace("/", "_").strip(".") # runs after every scenario @@ -189,22 +160,14 @@ def hook(context): clear_waited_after_sync() close_socket_connection() - # capture a screenshot if there is error or test failure in the current scenario execution - if scenario_failed() and os.getenv("CI") and is_linux(): - # scenario name can have "/" which is invalid filename - filename = get_screenshot_name(context.title) - directory = os.path.join(get_config("guiTestReportDir"), "screenshots") - if not os.path.exists(directory): - os.makedirs(directory) - try: - squish.saveDesktopScreenshot(os.path.join(directory, filename)) - except: - test.log("Failed to save screenshot") - - # check video report - if get_config("screenRecordOnFailure"): - filename = get_screenrecord_name(context.title) - save_screenrecord(filename) + # generate screenshot and video reports + if is_linux(): + filename = scenario_title_to_filename(context.title) + if scenario_failed(): + take_screenshot(f"{filename}.png") + + if get_config("video_recording_started"): + save_video_recording(f"{filename}.mp4", scenario_failed()) # teardown accounts and configs teardown_client() diff --git a/test/gui/shared/scripts/helpers/ConfigHelper.py b/test/gui/shared/scripts/helpers/ConfigHelper.py index a853737bb29..f3869692d21 100644 --- a/test/gui/shared/scripts/helpers/ConfigHelper.py +++ b/test/gui/shared/scripts/helpers/ConfigHelper.py @@ -73,12 +73,14 @@ def get_default_home_dir(): 'clientConfigDir': 'CLIENT_CONFIG_DIR', 'guiTestReportDir': 'GUI_TEST_REPORT_DIR', 'ocis': 'OCIS', - 'screenRecordOnFailure': 'SCREEN_RECORD_ON_FAILURE', + 'record_video_on_failure': 'RECORD_VIDEO_ON_FAILURE', } DEFAULT_PATH_CONFIG = { 'custom_lib': os.path.abspath('../shared/scripts/custom_lib'), 'home_dir': get_default_home_dir(), + # allow to record first 5 videos + 'video_record_limit': 5, } # default config values @@ -95,12 +97,16 @@ def get_default_home_dir(): 'clientConfigDir': get_config_home(), 'guiTestReportDir': os.path.abspath('../reports'), 'ocis': False, - 'screenRecordOnFailure': False, + 'record_video_on_failure': False, + 'retrying': False, + 'video_recording_started': False, } CONFIG.update(DEFAULT_PATH_CONFIG) READONLY_CONFIG = list(CONFIG_ENV_MAP.keys()) + list(DEFAULT_PATH_CONFIG.keys()) +SCENARIO_CONFIGS = {} + def read_cfg_file(cfg_path): cfg = ConfigParser() @@ -108,7 +114,7 @@ def read_cfg_file(cfg_path): for key, _ in CONFIG.items(): if key in CONFIG_ENV_MAP: if value := cfg.get('DEFAULT', CONFIG_ENV_MAP[key]): - if key in ('ocis', 'screenRecordOnFailure'): + if key in ('ocis', 'record_video_on_failure'): CONFIG[key] = value == 'true' else: CONFIG[key] = value @@ -128,7 +134,7 @@ def init_config(): # read and override configs from environment variables for key, value in CONFIG_ENV_MAP.items(): if os.environ.get(value): - if key in ('ocis', 'screenRecordOnFailure'): + if key in ('ocis', 'record_video_on_failure'): CONFIG[key] = os.environ.get(value) == 'true' else: CONFIG[key] = os.environ.get(value) @@ -154,22 +160,19 @@ def init_config(): CONFIG[key] = value.rstrip('/') + '/' -def get_config(key=None): - if key: - return CONFIG[key] - return CONFIG +def get_config(key): + return CONFIG[key] def set_config(key, value): if key in READONLY_CONFIG: raise KeyError(f'Cannot set read-only config: {key}') + # save the initial config value + if key not in SCENARIO_CONFIGS: + SCENARIO_CONFIGS[key] = CONFIG.get(key) CONFIG[key] = value def clear_scenario_config(): - global CONFIG - initial_config = {} - for key in READONLY_CONFIG: - initial_config[key] = CONFIG[key] - - CONFIG = initial_config + for key, value in SCENARIO_CONFIGS.items(): + CONFIG[key] = value diff --git a/test/gui/shared/scripts/helpers/ReportHelper.py b/test/gui/shared/scripts/helpers/ReportHelper.py new file mode 100644 index 00000000000..923ccff761d --- /dev/null +++ b/test/gui/shared/scripts/helpers/ReportHelper.py @@ -0,0 +1,75 @@ +import os +import glob +import shutil +import test +import squish +import squishinfo + +from helpers.ConfigHelper import get_config +from helpers.FilesHelper import prefix_path_namespace + + +def get_screenrecords_path(): + return os.path.join(get_config("guiTestReportDir"), "screenrecords") + + +def get_screenshots_path(): + return os.path.join(get_config("guiTestReportDir"), "screenshots") + + +def is_video_enabled(): + return ( + get_config("record_video_on_failure") + or get_config("retrying") + and not reached_video_limit() + ) + + +def reached_video_limit(): + video_report_dir = get_screenrecords_path() + if not os.path.exists(video_report_dir): + return False + entries = [f for f in os.scandir(video_report_dir) if f.is_file()] + return len(entries) >= get_config("video_record_limit") + + +def save_video_recording(filename, test_failed): + try: + # do not throw if stopVideoCapture() fails + test.stopVideoCapture() + except: + test.log("Failed to stop screen recording") + + if not (video_dir := squishinfo.resultDir): + video_dir = squishinfo.testCase + else: + test_case = "/".join(squishinfo.testCase.split("/")[-2:]) + video_dir = os.path.join(video_dir, test_case) + video_dir = os.path.join(video_dir, "attachments") + + # if the test failed + # move videos to the screenrecords directory + if test_failed: + video_files = glob.glob(f"{video_dir}/**/*.mp4", recursive=True) + screenrecords_dir = get_screenrecords_path() + if not os.path.exists(screenrecords_dir): + os.makedirs(screenrecords_dir) + # reverse the list to get the latest video first + video_files.reverse() + for idx, video in enumerate(video_files): + if idx: + file_parts = filename.rsplit(".", 1) + filename = f"{file_parts[0]}_{idx+1}.{file_parts[1]}" + shutil.move(video, os.path.join(screenrecords_dir, filename)) + # remove the video directory + shutil.rmtree(prefix_path_namespace(video_dir)) + + +def take_screenshot(filename): + directory = get_screenshots_path() + if not os.path.exists(directory): + os.makedirs(directory) + try: + squish.saveDesktopScreenshot(os.path.join(directory, filename)) + except: + test.log("Failed to save screenshot") diff --git a/test/gui/shared/scripts/helpers/SetupClientHelper.py b/test/gui/shared/scripts/helpers/SetupClientHelper.py index ea6ebf2ed9d..6ec1cc57485 100644 --- a/test/gui/shared/scripts/helpers/SetupClientHelper.py +++ b/test/gui/shared/scripts/helpers/SetupClientHelper.py @@ -13,6 +13,7 @@ from helpers.SyncHelper import listen_sync_status_for_item from helpers.api.utils import url_join from helpers.UserHelper import get_displayname_for_user +from helpers.ReportHelper import is_video_enabled def substitute_inline_codes(value): @@ -103,8 +104,9 @@ def start_client(): + ' --logdebug' + ' --logflush' ) - if get_config('screenRecordOnFailure'): + if is_video_enabled(): test.startVideoCapture() + set_config('video_recording_started', True) def get_polling_interval():