From 339a3a5ca9dd8fb7f14340303f0bf869c0f8d3e6 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Thu, 1 Jul 2021 17:03:39 -0500 Subject: [PATCH 1/5] WIP: add helper functions for resource prefixes --- test_utils/prefix.py | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 test_utils/prefix.py diff --git a/test_utils/prefix.py b/test_utils/prefix.py new file mode 100644 index 0000000..fd83e12 --- /dev/null +++ b/test_utils/prefix.py @@ -0,0 +1,43 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import datetime +import random + + +RESOURCE_PREFIX = "python_bigquery_datatransfer_samples_snippets" +RESOURCE_DATE_FORMAT = "%Y%m%d%H%M%S" +RESOURCE_DATE_LENGTH = 4 + 2 + 2 + 2 + 2 + 2 + + +def resource_prefix() -> str: + timestamp = datetime.datetime.utcnow().strftime(RESOURCE_DATE_FORMAT) + random_string = hex(random.randrange(1000000))[2:] + return f"{RESOURCE_PREFIX}_{timestamp}_{random_string}" + + +def resource_name_to_date(resource_name: str) -> datetime.datetime: + start_date = len(RESOURCE_PREFIX) + 1 + date_string = resource_name[start_date : start_date + RESOURCE_DATE_LENGTH] + parsed_date = datetime.datetime.strptime(date_string, RESOURCE_DATE_FORMAT) + return parsed_date + + +def should_cleanup(resource_name: str) -> bool: + yesterday = datetime.datetime.utcnow() - datetime.timedelta(days=1) + return ( + resource_name.startswith(RESOURCE_PREFIX) + and resource_name_to_date(resource_name) < yesterday + ) + From 6db1c426c6237ef52b1df0fe881c606eccea5a45 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Fri, 2 Jul 2021 16:45:35 -0500 Subject: [PATCH 2/5] add tests --- test_utils/prefix.py | 43 ------------------ test_utils/prefixer.py | 78 ++++++++++++++++++++++++++++++++ tests/unit/test_prefixer.py | 89 +++++++++++++++++++++++++++++++++++++ 3 files changed, 167 insertions(+), 43 deletions(-) delete mode 100644 test_utils/prefix.py create mode 100644 test_utils/prefixer.py create mode 100644 tests/unit/test_prefixer.py diff --git a/test_utils/prefix.py b/test_utils/prefix.py deleted file mode 100644 index fd83e12..0000000 --- a/test_utils/prefix.py +++ /dev/null @@ -1,43 +0,0 @@ -# Copyright 2021 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import datetime -import random - - -RESOURCE_PREFIX = "python_bigquery_datatransfer_samples_snippets" -RESOURCE_DATE_FORMAT = "%Y%m%d%H%M%S" -RESOURCE_DATE_LENGTH = 4 + 2 + 2 + 2 + 2 + 2 - - -def resource_prefix() -> str: - timestamp = datetime.datetime.utcnow().strftime(RESOURCE_DATE_FORMAT) - random_string = hex(random.randrange(1000000))[2:] - return f"{RESOURCE_PREFIX}_{timestamp}_{random_string}" - - -def resource_name_to_date(resource_name: str) -> datetime.datetime: - start_date = len(RESOURCE_PREFIX) + 1 - date_string = resource_name[start_date : start_date + RESOURCE_DATE_LENGTH] - parsed_date = datetime.datetime.strptime(date_string, RESOURCE_DATE_FORMAT) - return parsed_date - - -def should_cleanup(resource_name: str) -> bool: - yesterday = datetime.datetime.utcnow() - datetime.timedelta(days=1) - return ( - resource_name.startswith(RESOURCE_PREFIX) - and resource_name_to_date(resource_name) < yesterday - ) - diff --git a/test_utils/prefixer.py b/test_utils/prefixer.py new file mode 100644 index 0000000..fee06f9 --- /dev/null +++ b/test_utils/prefixer.py @@ -0,0 +1,78 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import datetime +import random +import re + + +_RESOURCE_DATE_FORMAT = "%Y%m%d%H%M%S" +_RESOURCE_DATE_LENGTH = 4 + 2 + 2 + 2 + 2 + 2 +_RE_SEPARATORS = re.compile(r"[/\-\\_]") + + +def _common_prefix(repo, relative_dir, separator="_"): + repo = re.sub(_RE_SEPARATORS, separator, repo) + relative_dir = re.sub(_RE_SEPARATORS, separator, relative_dir) + return f"{repo}{separator}{relative_dir}" + + +class Prefixer(object): + """ + Usage: + + Creating resources: + + >>> import test_utils.prefixer + >>> prefixer = test_utils.prefixer.Prefixer("python-bigquery", "samples/snippets") + >>> dataset_id = prefixer.create_prefix() + "my_sample" + + Cleaning up resources: + + >>> @pytest.fixture(scope="session", autouse=True) + ... def cleanup_datasets(bigquery_client: bigquery.Client): + ... for dataset in bigquery_client.list_datasets(): + ... if prefixer.should_cleanup(dataset.dataset_id): + ... bigquery_client.delete_dataset( + ... dataset, delete_contents=True, not_found_ok=True + """ + + def __init__( + self, repo, relative_dir, separator="_", cleanup_age=datetime.timedelta(days=1) + ): + self._separator = separator + self._cleanup_age = cleanup_age + self._prefix = _common_prefix(repo, relative_dir, separator=separator) + + def create_prefix(self) -> str: + timestamp = datetime.datetime.utcnow().strftime(_RESOURCE_DATE_FORMAT) + random_string = hex(random.randrange(0x1000000))[2:] + return f"{self._prefix}{self._separator}{timestamp}{self._separator}{random_string}" + + def _name_to_date(self, resource_name: str) -> datetime.datetime: + start_date = len(self._prefix) + len(self._separator) + date_string = resource_name[start_date : start_date + _RESOURCE_DATE_LENGTH] + try: + parsed_date = datetime.datetime.strptime(date_string, _RESOURCE_DATE_FORMAT) + return parsed_date + except ValueError: + return None + + def should_cleanup(self, resource_name: str) -> bool: + yesterday = datetime.datetime.utcnow() - self._cleanup_age + if not resource_name.startswith(self._prefix): + return False + + created_date = self._name_to_date(resource_name) + return created_date is not None and created_date < yesterday diff --git a/tests/unit/test_prefixer.py b/tests/unit/test_prefixer.py new file mode 100644 index 0000000..37157cc --- /dev/null +++ b/tests/unit/test_prefixer.py @@ -0,0 +1,89 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import datetime +import re + +import pytest + +import test_utils.prefixer + + +class FakeDateTime(object): + """Fake datetime class since pytest can't monkeypatch attributes of + built-in/extension type. + """ + + def __init__(self, fake_now): + self._fake_now = fake_now + + def utcnow(self): + return self._fake_now + + strptime = datetime.datetime.strptime + + +@pytest.mark.parametrize( + ("repo", "relative_dir", "separator", "expected"), + [ + ( + "python-bigquery", + "samples/snippets", + "_", + "python_bigquery_samples_snippets", + ), + ("python-storage", "samples\\snippets", "-", "python-storage-samples-snippets"), + ], +) +def test_common_prefix(repo, relative_dir, separator, expected): + got = test_utils.prefixer._common_prefix(repo, relative_dir, separator=separator) + assert got == expected + + +def test_create_prefix(monkeypatch): + fake_datetime = FakeDateTime(datetime.datetime(2021, 6, 21, 3, 32, 0)) + monkeypatch.setattr(datetime, "datetime", fake_datetime) + + prefixer = test_utils.prefixer.Prefixer( + "python-test-utils", "tests/unit", separator="?" + ) + got = prefixer.create_prefix() + parts = got.split("?") + assert len(parts) == 7 + assert "?".join(parts[:5]) == "python?test?utils?tests?unit" + datetime_part = parts[5] + assert datetime_part == "20210621033200" + random_hex_part = parts[6] + assert re.fullmatch("[0-9a-f]+", random_hex_part) + + +@pytest.mark.parametrize( + ("resource_name", "separator", "expected"), + [ + ("test_utils_created_elsewhere", "_", False), + ("test_utils_20210620120000", "_", False), + ("test_utils_20210620120000_abcdef_my_name", "_", False), + ("test_utils_20210619120000", "_", True), + ("test_utils_20210619120000_abcdef_my_name", "_", True), + ("test?utils?created?elsewhere", "_", False), + ("test?utils?20210620120000", "?", False), + ("test?utils?20210619120000", "?", True), + ], +) +def test_should_cleanup(resource_name, separator, expected, monkeypatch): + fake_datetime = FakeDateTime(datetime.datetime(2021, 6, 21, 3, 32, 0)) + monkeypatch.setattr(datetime, "datetime", fake_datetime) + + prefixer = test_utils.prefixer.Prefixer("test", "utils", separator=separator) + assert prefixer.should_cleanup(resource_name) == expected From 232deb810dd6edeb69751d8613f60aa4960f01f1 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Wed, 7 Jul 2021 09:12:20 -0500 Subject: [PATCH 3/5] use compiled sub method Co-authored-by: Tres Seaver --- test_utils/prefixer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test_utils/prefixer.py b/test_utils/prefixer.py index fee06f9..53e5161 100644 --- a/test_utils/prefixer.py +++ b/test_utils/prefixer.py @@ -23,8 +23,8 @@ def _common_prefix(repo, relative_dir, separator="_"): - repo = re.sub(_RE_SEPARATORS, separator, repo) - relative_dir = re.sub(_RE_SEPARATORS, separator, relative_dir) + repo = _RE_SEPARATORS.sub(separator, repo) + relative_dir = _RE_SEPARATORS.sub(separator, relative_dir) return f"{repo}{separator}{relative_dir}" From e55c0e07169a67cdbe813b48e22942301c8f1640 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Wed, 7 Jul 2021 09:12:38 -0500 Subject: [PATCH 4/5] add summary docstring Co-authored-by: Tres Seaver --- test_utils/prefixer.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test_utils/prefixer.py b/test_utils/prefixer.py index 53e5161..4454018 100644 --- a/test_utils/prefixer.py +++ b/test_utils/prefixer.py @@ -29,7 +29,8 @@ def _common_prefix(repo, relative_dir, separator="_"): class Prefixer(object): - """ + """Create/manage resource IDs for system testing. + Usage: Creating resources: From d91771e174afc309a204b060ff27cf05f463aec6 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Wed, 7 Jul 2021 09:26:48 -0500 Subject: [PATCH 5/5] clean whitespace --- test_utils/prefixer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_utils/prefixer.py b/test_utils/prefixer.py index 4454018..6d85867 100644 --- a/test_utils/prefixer.py +++ b/test_utils/prefixer.py @@ -30,7 +30,7 @@ def _common_prefix(repo, relative_dir, separator="_"): class Prefixer(object): """Create/manage resource IDs for system testing. - + Usage: Creating resources: