From 89fa5ccf5086e9647ed907d328601fd94dd78908 Mon Sep 17 00:00:00 2001 From: John Spray Date: Mon, 26 Jun 2023 11:14:09 +0100 Subject: [PATCH 1/3] tests: small refactor in describe_topic This function was building a list, and then iter-izing in during return. Since we return an iterator, we might as well just yield as we go. --- tests/rptest/clients/rpk.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/rptest/clients/rpk.py b/tests/rptest/clients/rpk.py index 19937826703a..1b54591d004d 100644 --- a/tests/rptest/clients/rpk.py +++ b/tests/rptest/clients/rpk.py @@ -406,7 +406,6 @@ def int_or_none(value): self._redpanda.logger.error(f"Missing columns: {missing_columns}") raise RpkException(f"Missing columns: {missing_columns}") - partitions = [] for row in table.rows: obj = dict() obj["LAST-STABLE-OFFSET"] = "-" @@ -440,9 +439,7 @@ def int_or_none(value): start_offset=obj["LOG-START-OFFSET"]) if initialized or tolerant: - partitions.append(partition) - - return iter(partitions) + yield partition def describe_topic_configs(self, topic): cmd = ['describe', topic, '-c'] From 37b2dcd707504aec93c750479ae8beab37c87ee8 Mon Sep 17 00:00:00 2001 From: John Spray Date: Mon, 26 Jun 2023 11:14:47 +0100 Subject: [PATCH 2/3] tests: improve quiesce_uploads This function could get zero partitions from its call to `rpk topic describe`, and would proceed immediately without waiting for anything. We expect to always be called for a topic that exists, so throw an exception if describing the topic doesn't get us any partitions. --- tests/rptest/utils/si_utils.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/rptest/utils/si_utils.py b/tests/rptest/utils/si_utils.py index 9dc199abd91f..4e3c7560ca5e 100644 --- a/tests/rptest/utils/si_utils.py +++ b/tests/rptest/utils/si_utils.py @@ -466,7 +466,10 @@ def remote_has_reached_hwm(ntp: NTP, hwm: int): rpk = RpkTool(redpanda) for topic_name in topic_names: described = rpk.describe_topic(topic_name) + p_count = 0 for p in described: + p_count += 1 + ntp = NTP(ns='kafka', topic=topic_name, partition=p.id) hwm = p.high_watermark @@ -479,6 +482,11 @@ def remote_has_reached_hwm(ntp: NTP, hwm: int): backoff_sec=1) redpanda.logger.debug(f"Partition {ntp} ready (reached HWM {hwm})") + if p_count == 0: + # We expect to be called on a topic where `rpk topic describe` returns + # some data, otherwise we can't check that against cloud storage content + raise RuntimeError(f"Found 0 partitions for topic '{topic_name}'") + @dataclass(order=True) class SpillMeta: From 5f8fd9fe8480c090ad3fa02d5a606f72efe820c3 Mon Sep 17 00:00:00 2001 From: John Spray Date: Mon, 26 Jun 2023 11:16:09 +0100 Subject: [PATCH 3/3] tests: fix quiesce call in topic deletion test This fixes commit 8355e4ec4593c852401c3ade97871da52d5e04b7 quiesce_uploads would fail silently if you called it for a topic that didn't exist: that's fixed in the previous commit. This commit fixes the underlying issue, that we were passing a string instead of a list of strings, so quiesce_upload was trying to wait for each character in the string as if it was a topic name. Fixes https://github.com/redpanda-data/redpanda/issues/8496 Fixes https://github.com/redpanda-data/redpanda/issues/10655 Fixes https://github.com/redpanda-data/redpanda/issues/9629 --- tests/rptest/tests/topic_delete_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/rptest/tests/topic_delete_test.py b/tests/rptest/tests/topic_delete_test.py index 98b97a1351c0..066d97a37c1b 100644 --- a/tests/rptest/tests/topic_delete_test.py +++ b/tests/rptest/tests/topic_delete_test.py @@ -472,7 +472,7 @@ def _populate_topic(self, # Wait for everything to be uploaded: this avoids tests potentially trying # to delete topics mid-uploads, which can leave orphan segments. - quiesce_uploads(self.redpanda, topic_name, timeout_sec=60) + quiesce_uploads(self.redpanda, [topic_name], timeout_sec=60) @skip_debug_mode # Rely on timely uploads during leader transfers @cluster(num_nodes=3,