From 906a4efb2498e064d8c789ecb057e0aa86ca5bd3 Mon Sep 17 00:00:00 2001 From: Jan Honsbrok Date: Wed, 10 Nov 2021 20:28:36 +0100 Subject: [PATCH] migrate update reference image commands tests away from pytest-bdd --- tests/integrationtests/cato_cmd/conftest.py | 33 ---- .../cato_cmd/test_run_command.feature | 19 -- ...st_update_missing_reference_images.feature | 13 -- .../test_update_reference_image.feature | 25 --- .../cato_cmd/test_update_reference_image.py | 180 +++++++----------- 5 files changed, 74 insertions(+), 196 deletions(-) delete mode 100644 tests/integrationtests/cato_cmd/test_run_command.feature delete mode 100644 tests/integrationtests/cato_cmd/test_update_missing_reference_images.feature delete mode 100644 tests/integrationtests/cato_cmd/test_update_reference_image.feature diff --git a/tests/integrationtests/cato_cmd/conftest.py b/tests/integrationtests/cato_cmd/conftest.py index f492632b..e1e67dfc 100644 --- a/tests/integrationtests/cato_cmd/conftest.py +++ b/tests/integrationtests/cato_cmd/conftest.py @@ -5,13 +5,6 @@ import pytest -@pytest.fixture -def dir_changer(): - old_dir = os.getcwd() - yield - os.chdir(old_dir) - - @pytest.fixture def cato_config(tmp_path, test_resource_provider): config_folder = os.path.join(str(tmp_path), "config_folder") @@ -32,29 +25,3 @@ def cato_config(tmp_path, test_resource_provider): with open(config_path, "wb") as f: f.write(content.encode()) return config_folder, config_path - - -@pytest.fixture -def create_result_path(cato_config): - config_folder, config_path = cato_config - - def f(suite_name, test_name): - return os.path.join( - config_folder, - "result", - suite_name, - test_name, - test_name + ".png", - ) - - return f - - -@pytest.fixture -def create_reference_path(cato_config): - config_folder, config_path = cato_config - - def f(suite_name, test_name): - return os.path.join(config_folder, suite_name, test_name, "reference.png") - - return f diff --git a/tests/integrationtests/cato_cmd/test_run_command.feature b/tests/integrationtests/cato_cmd/test_run_command.feature deleted file mode 100644 index bd9e58f2..00000000 --- a/tests/integrationtests/cato_cmd/test_run_command.feature +++ /dev/null @@ -1,19 +0,0 @@ -Feature: Running tests described in a config file - - Scenario: Run all tests the first time should fail because of missing reference images - Given a cato.json file with tests - And no reference images for the tests - When I run the run command - Then All tests should have been executed - And A failure message should be printed - And Failure Messages for missing reference image should be printed - And the result should be available at the server - - Scenario: Run all tests should succeed - Given a cato.json file with tests - And reference images exist for the tests - When I run the run command - Then All tests should have been executed - And no failure message should be printed - And a success message should be printed - And the success result should be available at the server \ No newline at end of file diff --git a/tests/integrationtests/cato_cmd/test_update_missing_reference_images.feature b/tests/integrationtests/cato_cmd/test_update_missing_reference_images.feature deleted file mode 100644 index f1040aee..00000000 --- a/tests/integrationtests/cato_cmd/test_update_missing_reference_images.feature +++ /dev/null @@ -1,13 +0,0 @@ -# Created by Jan at 21.03.2021 -Feature: Updating missing reference images - Updates the reference images for tests without one to the last test output. - Useful for adding a baseline - - Scenario: Updating missing reference image - Given a cato.json file with tests - And no reference image for a test - And an output image for a test - And reference images for other tests - When I run the update missing reference images command - Then the output image should be stored as reference image - And other reference images should be untouched \ No newline at end of file diff --git a/tests/integrationtests/cato_cmd/test_update_reference_image.feature b/tests/integrationtests/cato_cmd/test_update_reference_image.feature deleted file mode 100644 index 76d616de..00000000 --- a/tests/integrationtests/cato_cmd/test_update_reference_image.feature +++ /dev/null @@ -1,25 +0,0 @@ -Feature: Updating a reference image - Updating a reference image to a new one (e.g. add a new baseline) - - Scenario: Updating reference image which did not exist before - Given a cato.json file with tests - And an output image for a test - And reference images for other tests - When I run the update reference image command - Then the output image should be stored as reference image - And other reference images should be untouched - - Scenario: Updating reference image which did exist before - Given a cato.json file with tests - And an output image for a test - And reference images for other tests - And a reference image exists for the test - When I run the update reference image command - Then the output image should be stored as reference image - And other reference images should be untouched - - Scenario: Updating reference to a not existing output file fails - Given a cato.json file with tests - And reference images for other tests - When I run the update reference image command for a not existing test - Then all reference images should be untouched \ No newline at end of file diff --git a/tests/integrationtests/cato_cmd/test_update_reference_image.py b/tests/integrationtests/cato_cmd/test_update_reference_image.py index 647a6d47..645d3682 100644 --- a/tests/integrationtests/cato_cmd/test_update_reference_image.py +++ b/tests/integrationtests/cato_cmd/test_update_reference_image.py @@ -8,150 +8,118 @@ from tests.integrationtests.command_fixture import run_cato_command -@scenario( - "test_update_reference_image.feature", - "Updating reference image which did not exist before", -) -def test_update_not_existing_before(): - pass - - -@scenario( - "test_update_reference_image.feature", - "Updating reference image which did exist before", -) -def test_update_existing(): - pass +def test_updating_reference_image_which_did_not_exist_before(cato_config): + output_image_path, output_image_hash = ensure_image_output_for_a_test(cato_config) + run_update_reference_image_command(cato_config) + assert_output_image_should_be_stored_as_reference_image( + cato_config, output_image_hash + ) -@scenario( - "test_update_reference_image.feature", - "Updating reference to a not existing output file fails", -) -def test_update_not_existing_test(): - pass +def test_updating_missing_reference_images(cato_config): + output_image_path, output_image_hash = ensure_image_output_for_a_test(cato_config) + ensure_reference_image_for_other_test(cato_config) + run_update_missing_reference_image_command(cato_config) + ensure_output_image_is_stored_as_reference_image(cato_config, output_image_hash) -@scenario( - "test_update_missing_reference_images.feature", - "Updating missing reference image", -) -def test_update_not_existing(): - pass +def run_update_reference_image_command(cato_config): + config_folder, config_path = cato_config + os.chdir(config_folder) + command_result = run_cato_command( + ["update-reference", "--test-identifier", "WriteImages/write_white_image"] + ) -@given("a cato.json file with tests") -def step_impl(cato_config): - pass + assert command_result.exit_code == 0 -def write_image(path, color=None): - folder = os.path.dirname(path) - if not os.path.exists(folder): - os.makedirs(folder) - img = Image.new("RGB", (800, 600), (255, 255, 255) if not color else color) - img.save(path, "PNG") +def ensure_image_output_for_a_test(cato_config): + image_path = create_result_path(cato_config, "WriteImages", "write_white_image") + write_image(image_path) + return image_path, hash_file(image_path) -def hash_file(path): - return hashlib.md5(open(path, "rb").read()).hexdigest() +def create_result_path(cato_config, suite_name, test_name): + config_folder, config_path = cato_config + return os.path.join( + config_folder, + "result", + suite_name, + test_name, + test_name + ".png", + ) -def hash_directory(path): - return dirhash(path, "md5") +def ensure_reference_image_for_other_test(cato_config): + config_folder, config_path = cato_config + reference_image_folder = os.path.join(config_folder, "WriteImages") + image_path = create_reference_path(cato_config, "WriteImages", "write_black_image") + write_image(image_path, (0, 0, 0)) -@given("an output image for a test") -def step_impl(scenario_context, create_result_path): - image_path = create_result_path("WriteImages", "write_white_image") - write_image(image_path) - scenario_context["output_image"] = image_path - scenario_context["output_image_checksum"] = hash_file(image_path) - + other_tests_reference_image = image_path + other_tests_reference_checksum = hash_file(image_path) + reference_image_folder = reference_image_folder + reference_dir_checksum = hash_directory(reference_image_folder) -@given("reference images for other tests") -def step_impl(scenario_context, create_reference_path): - reference_image_folder = os.path.join( - scenario_context["config_folder"], "WriteImages" - ) - image_path = create_reference_path("WriteImages", "write_black_image") - write_image(image_path, (0, 0, 0)) +def create_reference_path(cato_config, suite_name, test_name): + config_folder, config_path = cato_config - scenario_context["other_tests_reference_image"] = image_path - scenario_context["other_tests_reference_checksum"] = hash_file(image_path) - scenario_context["reference_image_folder"] = reference_image_folder - scenario_context["reference_dir_checksum"] = hash_directory(reference_image_folder) + return os.path.join(config_folder, suite_name, test_name, "reference.png") -@given("a reference image exists for the test") -def step_impl(scenario_context, create_reference_path): - image_path = create_reference_path("WriteImages", "write_white_image") +def assert_output_image_should_be_stored_as_reference_image( + cato_config, output_image_checksum +): + reference_image_path = create_reference_path( + cato_config, "WriteImages", "write_white_image" + ) - write_image(image_path, (1, 1, 1)) + assert os.path.exists(reference_image_path) - scenario_context["test_reference_image"] = image_path - scenario_context["test_reference_checksum"] = hash_file(image_path) + reference_image_checksum = hash_file(reference_image_path) + assert output_image_checksum == reference_image_checksum -@when("I run the update reference image command") -def step_impl(scenario_context, dir_changer): - os.chdir(scenario_context["config_folder"]) +def run_update_missing_reference_image_command(cato_config): + config_folder, config_path = cato_config + os.chdir(config_folder) command_result = run_cato_command( - ["update-reference", "--test-identifier", "WriteImages/write_white_image"] + [ + "update-missing-reference-images", + ] ) assert command_result.exit_code == 0 -@when("I run the update reference image command for a not existing test") -def step_impl(scenario_context, dir_changer): - os.chdir(scenario_context["config_folder"]) - - command_result = run_cato_command( - ["update-reference", "--test-identifier", "WriteImages/NOT_EXISTING"] +def ensure_output_image_is_stored_as_reference_image( + cato_config, output_image_checksum +): + reference_image_path = create_reference_path( + cato_config, "WriteImages", "write_white_image" ) - -@then("the output image should be stored as reference image") -def step_impl(scenario_context, create_reference_path): - reference_image_path = create_reference_path("WriteImages", "write_white_image") - assert os.path.exists(reference_image_path) reference_image_checksum = hash_file(reference_image_path) - assert scenario_context["output_image_checksum"] == reference_image_checksum - + assert output_image_checksum == reference_image_checksum -@then("other reference images should be untouched") -def step_impl(scenario_context): - assert ( - hash_file(scenario_context["other_tests_reference_image"]) - == scenario_context["other_tests_reference_checksum"] - ) - - -@then("all reference images should be untouched") -def step_impl(scenario_context): - assert scenario_context["reference_dir_checksum"] == hash_directory( - scenario_context["reference_image_folder"] - ) - -@given("no reference image for a test") -def step_impl(): - pass +def write_image(path, color=None): + folder = os.path.dirname(path) + if not os.path.exists(folder): + os.makedirs(folder) + img = Image.new("RGB", (800, 600), (255, 255, 255) if not color else color) + img.save(path, "PNG") -@when("I run the update missing reference images command") -def step_impl(scenario_context, dir_changer): - os.chdir(scenario_context["config_folder"]) +def hash_file(path): + return hashlib.md5(open(path, "rb").read()).hexdigest() - command_result = run_cato_command( - [ - "update-missing-reference-images", - ] - ) - assert command_result.exit_code == 0 +def hash_directory(path): + return dirhash(path, "md5")