From 92464b28979ae1eda19be4141c18603c5fafe410 Mon Sep 17 00:00:00 2001 From: Tobias Grigo Date: Mon, 5 Feb 2024 16:19:10 +0100 Subject: [PATCH] Improve performance test to save test results in csv files [noissue] --- pulp_deb/tests/performance/test_publish.py | 54 ++++++++++------ .../tests/performance/test_pulp_to_pulp.py | 51 +++++++++------- pulp_deb/tests/performance/test_sync.py | 61 +++++++++++++------ pulp_deb/tests/performance/utils.py | 23 +++++++ 4 files changed, 129 insertions(+), 60 deletions(-) create mode 100644 pulp_deb/tests/performance/utils.py diff --git a/pulp_deb/tests/performance/test_publish.py b/pulp_deb/tests/performance/test_publish.py index fb4bb6283..63694083a 100644 --- a/pulp_deb/tests/performance/test_publish.py +++ b/pulp_deb/tests/performance/test_publish.py @@ -1,6 +1,7 @@ """Tests that publish deb plugin repositories.""" import pytest +from pulp_deb.tests.performance.utils import write_csv_to_tmp from pulp_deb.tests.functional.constants import ( DEB_PACKAGE_NAME, DEB_PERF_BOOKWORN, @@ -9,14 +10,23 @@ DEB_PERF_UBUNTU_URL, ) +perf_publish_test_params = [ + pytest.param( + DEB_PERF_UBUNTU_URL, + DEB_PERF_JAMMY, + "pulp-deb-performance-publish-tests-ubuntu-jammy", + id="performance-publish-ubuntu-jammy", + ), + pytest.param( + DEB_PERF_DEBIAN_URL, + DEB_PERF_BOOKWORN, + "pulp-deb-performance-publish-tests-debian-bookworm", + id="performance-publish-debian-bookworm", + ), +] -@pytest.mark.parametrize( - "url,remote_args", - [ - (DEB_PERF_UBUNTU_URL, DEB_PERF_JAMMY), - (DEB_PERF_DEBIAN_URL, DEB_PERF_BOOKWORN), - ], -) + +@pytest.mark.parametrize("url,remote_args,csv_filename", perf_publish_test_params) def test_deb_publish( apt_publication_api, apt_repository_versions_api, @@ -24,17 +34,14 @@ def test_deb_publish( monitor_task, url, remote_args, + csv_filename, delete_orphans_pre, ): """Publish repositories with the deb plugin.""" repo, _, task = deb_init_and_sync(url=url, remote_args=remote_args, return_task=True) - task_duration = task.finished_at - task.started_at - waiting_time = task.started_at - task.pulp_created - print( - "\n-> Sync => Waiting time (s): {wait} | Service time (s): {service}".format( - wait=waiting_time.total_seconds(), service=task_duration.total_seconds() - ) - ) + task_time = (task.finished_at - task.started_at).total_seconds() + wait_time = (task.started_at - task.pulp_created).total_seconds() + print(f"\n-> Sync => Waiting time (s): {wait_time} | Service time (s): {task_time}") # Check that we have the correct content counts repo_ver = apt_repository_versions_api.read(repo.latest_version_href) @@ -44,10 +51,19 @@ def test_deb_publish( # Publishing response = apt_publication_api.create({"repository": repo.pulp_href}) task = monitor_task(response.task) - task_duration = task.finished_at - task.started_at - waiting_time = task.started_at - task.pulp_created + task_time_publish = (task.finished_at - task.started_at).total_seconds() + wait_time_publish = (task.started_at - task.pulp_created).total_seconds() print( - "\n-> Publish => Waiting time (s): {wait} | Service time (s): {service}".format( - wait=waiting_time.total_seconds(), service=task_duration.total_seconds() - ) + f"\n-> Publish => Wait time (s): {wait_time_publish} | Task time (s): {task_time_publish}" + ) + + write_csv_to_tmp( + csv_filename, + [ + "task_duration_sync", + "waiting_time_sync", + "task_duration_publish", + "waiting_time_publish", + ], + [task_time, wait_time, task_time_publish, wait_time_publish], ) diff --git a/pulp_deb/tests/performance/test_pulp_to_pulp.py b/pulp_deb/tests/performance/test_pulp_to_pulp.py index 0383e4163..38de61cbb 100644 --- a/pulp_deb/tests/performance/test_pulp_to_pulp.py +++ b/pulp_deb/tests/performance/test_pulp_to_pulp.py @@ -1,6 +1,7 @@ """Tests that verify download of deb content served by Pulp.""" import pytest +from pulp_deb.tests.performance.utils import write_csv_to_tmp from pulp_deb.tests.functional.utils import get_counts_from_content_summary from pulp_deb.tests.functional.constants import ( DEB_PACKAGE_NAME, @@ -10,15 +11,24 @@ DEB_PERF_UBUNTU_URL, ) +perf_p2p_test_params = [ + pytest.param( + DEB_PERF_UBUNTU_URL, + DEB_PERF_JAMMY, + "pulp-deb-p2p-tests-ubuntu-jammy", + id="performance-p2p-ubuntu-jammy", + ), + pytest.param( + DEB_PERF_DEBIAN_URL, + DEB_PERF_BOOKWORN, + "pulp-deb-p2p-tests-debian-bookworm", + id="performance-p2p-debian-bookworm", + ), +] + @pytest.mark.parallel -@pytest.mark.parametrize( - "url,remote_args", - [ - (DEB_PERF_UBUNTU_URL, DEB_PERF_JAMMY), - (DEB_PERF_DEBIAN_URL, DEB_PERF_BOOKWORN), - ], -) +@pytest.mark.parametrize("url,remote_args,csv_filename", perf_p2p_test_params) def test_pulp_to_pulp( deb_distribution_factory, deb_get_content_summary, @@ -26,16 +36,13 @@ def test_pulp_to_pulp( deb_publication_factory, url, remote_args, + csv_filename, ): """Verify whether content served by pulp can be synced.""" repo, _, task = deb_init_and_sync(url=url, remote_args=remote_args, return_task=True) - task_duration = task.finished_at - task.started_at - waiting_time = task.started_at - task.pulp_created - print( - "\n-> Sync => Waiting time (s): {wait} | Service time (s): {service}".format( - wait=waiting_time.total_seconds(), service=task_duration.total_seconds() - ) - ) + task_time = (task.finished_at - task.started_at).total_seconds() + wait_time = (task.started_at - task.pulp_created).total_seconds() + print(f"\n-> Sync => Waiting time (s): {wait_time} | Service time (s): {task_time}") # Create a publication and distribution publication = deb_publication_factory(repo) @@ -45,13 +52,9 @@ def test_pulp_to_pulp( repo2, _, task = deb_init_and_sync( url=distribution.base_url, remote_args=remote_args, return_task=True ) - task_duration = task.finished_at - task.started_at - waiting_time = task.started_at - task.pulp_created - print( - "\n-> Sync => Waiting time (s): {wait} | Service time (s): {service}".format( - wait=waiting_time.total_seconds(), service=task_duration.total_seconds() - ) - ) + task_time_p2p = (task.finished_at - task.started_at).total_seconds() + wait_time_p2p = (task.started_at - task.pulp_created).total_seconds() + print(f"\n-> Sync => Waiting time (s): {wait_time_p2p} | Service time (s): {task_time_p2p}") repo_summary = deb_get_content_summary(repo) repo2_summary = deb_get_content_summary(repo2) @@ -62,3 +65,9 @@ def test_pulp_to_pulp( added = get_counts_from_content_summary(repo_summary.added) added2 = get_counts_from_content_summary(repo2_summary.added) assert added[DEB_PACKAGE_NAME] == added2[DEB_PACKAGE_NAME] + + write_csv_to_tmp( + csv_filename, + ["task_duration_initial", "waiting_time_initial", "task_duration_p2p", "waiting_time_p2p"], + [task_time, wait_time, task_time_p2p, wait_time_p2p], + ) diff --git a/pulp_deb/tests/performance/test_sync.py b/pulp_deb/tests/performance/test_sync.py index 20c069340..2d5ced61d 100644 --- a/pulp_deb/tests/performance/test_sync.py +++ b/pulp_deb/tests/performance/test_sync.py @@ -1,6 +1,7 @@ """Tests that sync deb plugin repositories.""" import pytest +from pulp_deb.tests.performance.utils import write_csv_to_tmp from pulp_deb.tests.functional.constants import ( DEB_PERF_BOOKWORN, DEB_PERF_DEBIAN_URL, @@ -8,40 +9,60 @@ DEB_PERF_UBUNTU_URL, ) +perf_sync_test_params = [ + pytest.param( + DEB_PERF_UBUNTU_URL, + DEB_PERF_JAMMY, + True, + "pulp-deb-performance-sync-tests-ubuntu-jammy", + id="performance-sync-ubuntu-jammy-with-resync", + ), + pytest.param( + DEB_PERF_DEBIAN_URL, + DEB_PERF_BOOKWORN, + True, + "pulp-deb-performance-sync-tests-debian-bookworm", + id="performance-sync-debian-bookworm-with-resync", + ), +] + @pytest.mark.parallel -@pytest.mark.parametrize( - "url,remote_args,resync", - [ - (DEB_PERF_UBUNTU_URL, DEB_PERF_JAMMY, True), - (DEB_PERF_DEBIAN_URL, DEB_PERF_BOOKWORN, True), - ], -) -def test_deb_sync(deb_init_and_sync, url, remote_args, resync): +@pytest.mark.parametrize("url,remote_args,resync,csv_filename", perf_sync_test_params) +def test_deb_sync(deb_init_and_sync, url, remote_args, resync, csv_filename): """Sync repositories with the deb plugin.""" # Create repository & remote and sync repo, remote, task = deb_init_and_sync(url=url, remote_args=remote_args, return_task=True) - task_duration = task.finished_at - task.started_at - waiting_time = task.started_at - task.pulp_created - print( - "\n-> Sync => Waiting time (s): {wait} | Service time (s): {service}".format( - wait=waiting_time.total_seconds(), service=task_duration.total_seconds() - ) - ) + task_time = (task.finished_at - task.started_at).total_seconds() + wait_time = (task.started_at - task.pulp_created).total_seconds() + print(f"\n-> Sync => Waiting time (s): {wait_time} | Service time (s): {task_time}") if resync: # Sync the repository again. latest_version_href = repo.latest_version_href repo, _, task = deb_init_and_sync(repository=repo, remote=remote, return_task=True) - task_duration = task.finished_at - task.started_at - waiting_time = task.started_at - task.pulp_created + task_time_resync = (task.finished_at - task.started_at).total_seconds() + wait_time_resync = (task.started_at - task.pulp_created).total_seconds() print( - "\n-> Re-sync => Waiting time (s): {wait} | Service time (s): {service}".format( - wait=waiting_time.total_seconds(), service=task_duration.total_seconds() - ) + f"\n-> Resync => Wait time (s): {wait_time_resync} | Task time (s): {task_time_resync}" ) # Check that nothing has changed since the last sync. assert latest_version_href == repo.latest_version_href + + write_csv_to_tmp( + csv_filename, + [ + "task_duration_initial", + "waiting_time_initial", + "task_duration_resync", + "waiting_time_resync", + ], + [task_time, wait_time, task_time_resync, wait_time_resync], + ) + else: + write_csv_to_tmp( + csv_filename, ["task_duration_initial", "waiting_time_initial"], [task_time, wait_time] + ) diff --git a/pulp_deb/tests/performance/utils.py b/pulp_deb/tests/performance/utils.py new file mode 100644 index 000000000..812071fbd --- /dev/null +++ b/pulp_deb/tests/performance/utils.py @@ -0,0 +1,23 @@ +# coding=utf-8 +"""Utilities for testing deb performance""" +import csv + +from datetime import datetime + + +def write_csv_to_tmp(basename, header_row, content_row, add_date=True, dateformat="%d.%m.%y"): + """Write a CSV file with a header and a single content row to the /tmp/ folder. + + :param basename: The basename of the csv file. + :param header_row: The column header names. + :param content_row: The line of data for the content columns. + :param add_date: Whether the current date should be added to the csv. (default: True). + :param dateformat: How the date should be formatted (default: %d.%m.%y). + """ + with open(f"/tmp/{basename}.csv", "w", encoding="UTF-8", newline="") as f: + writer = csv.writer(f) + if add_date: + header_row.insert(0, "date") + content_row.insert(0, datetime.strftime(datetime.now(), dateformat)) + writer.writerow(header_row) + writer.writerow(content_row)