From d354349ccdbdb4e1af94ffe7c762c9d5507e58b2 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Mon, 11 Nov 2024 11:24:30 -0800 Subject: [PATCH] chore(tests): clean up sample test tables (#1030) --- samples/__init__.py | 0 samples/beam/__init__.py | 0 samples/beam/hello_world_write_test.py | 32 +- samples/hello/__init__.py | 0 samples/hello/async_main.py | 115 ++++--- samples/hello/async_main_test.py | 15 +- samples/hello/main.py | 116 ++++--- samples/hello/main_test.py | 15 +- samples/hello_happybase/__init__.py | 0 samples/hello_happybase/main.py | 6 +- samples/hello_happybase/main_test.py | 35 +- samples/quickstart/__init__.py | 0 samples/quickstart/main_async_test.py | 44 +-- samples/quickstart/main_test.py | 25 +- samples/quickstart_happybase/__init__.py | 0 samples/quickstart_happybase/main_test.py | 25 +- samples/snippets/__init__.py | 0 samples/snippets/data_client/__init__.py | 0 .../data_client_snippets_async_test.py | 26 +- samples/snippets/deletes/__init__.py | 0 .../snippets/deletes/deletes_async_test.py | 42 +-- samples/snippets/deletes/deletes_test.py | 125 ++++--- .../filters/filter_snippets_async_test.py | 38 +-- samples/snippets/filters/filters_test.py | 115 +++---- samples/snippets/reads/reads_test.py | 77 ++--- samples/snippets/writes/writes_test.py | 33 +- samples/tableadmin/__init__.py | 0 samples/tableadmin/tableadmin.py | 320 ++++++++---------- samples/tableadmin/tableadmin_test.py | 27 +- samples/utils.py | 87 +++++ 30 files changed, 605 insertions(+), 713 deletions(-) create mode 100644 samples/__init__.py create mode 100644 samples/beam/__init__.py create mode 100644 samples/hello/__init__.py create mode 100644 samples/hello_happybase/__init__.py create mode 100644 samples/quickstart/__init__.py create mode 100644 samples/quickstart_happybase/__init__.py create mode 100644 samples/snippets/__init__.py create mode 100644 samples/snippets/data_client/__init__.py create mode 100644 samples/snippets/deletes/__init__.py create mode 100644 samples/tableadmin/__init__.py create mode 100644 samples/utils.py diff --git a/samples/__init__.py b/samples/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/samples/beam/__init__.py b/samples/beam/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/samples/beam/hello_world_write_test.py b/samples/beam/hello_world_write_test.py index 4e9a47c7d..ba0e98096 100644 --- a/samples/beam/hello_world_write_test.py +++ b/samples/beam/hello_world_write_test.py @@ -14,45 +14,33 @@ import os import uuid -from google.cloud import bigtable import pytest -import hello_world_write +from . import hello_world_write +from ..utils import create_table_cm PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] BIGTABLE_INSTANCE = os.environ["BIGTABLE_INSTANCE"] -TABLE_ID_PREFIX = "mobile-time-series-{}" +TABLE_ID = f"mobile-time-series-beam-{str(uuid.uuid4())[:16]}" @pytest.fixture(scope="module", autouse=True) -def table_id(): - client = bigtable.Client(project=PROJECT, admin=True) - instance = client.instance(BIGTABLE_INSTANCE) +def table(): + with create_table_cm( + PROJECT, BIGTABLE_INSTANCE, TABLE_ID, {"stats_summary": None} + ) as table: + yield table - table_id = TABLE_ID_PREFIX.format(str(uuid.uuid4())[:16]) - table = instance.table(table_id) - if table.exists(): - table.delete() - table.create(column_families={"stats_summary": None}) - yield table_id - - table.delete() - - -def test_hello_world_write(table_id): +def test_hello_world_write(table): hello_world_write.run( [ "--bigtable-project=%s" % PROJECT, "--bigtable-instance=%s" % BIGTABLE_INSTANCE, - "--bigtable-table=%s" % table_id, + "--bigtable-table=%s" % TABLE_ID, ] ) - client = bigtable.Client(project=PROJECT, admin=True) - instance = client.instance(BIGTABLE_INSTANCE) - table = instance.table(table_id) - rows = table.read_rows() count = 0 for _ in rows: diff --git a/samples/hello/__init__.py b/samples/hello/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/samples/hello/async_main.py b/samples/hello/async_main.py index d608bb073..0130161e1 100644 --- a/samples/hello/async_main.py +++ b/samples/hello/async_main.py @@ -26,6 +26,7 @@ import argparse import asyncio +from ..utils import wait_for_table # [START bigtable_async_hw_imports] from google.cloud import bigtable @@ -33,7 +34,6 @@ from google.cloud.bigtable.data import RowMutationEntry from google.cloud.bigtable.data import SetCell from google.cloud.bigtable.data import ReadRowsQuery - # [END bigtable_async_hw_imports] @@ -65,63 +65,66 @@ async def main(project_id, instance_id, table_id): print("Table {} already exists.".format(table_id)) # [END bigtable_async_hw_create_table] - # [START bigtable_async_hw_write_rows] - print("Writing some greetings to the table.") - greetings = ["Hello World!", "Hello Cloud Bigtable!", "Hello Python!"] - mutations = [] - column = "greeting" - for i, value in enumerate(greetings): - # Note: This example uses sequential numeric IDs for simplicity, - # but this can result in poor performance in a production - # application. Since rows are stored in sorted order by key, - # sequential keys can result in poor distribution of operations - # across nodes. - # - # For more information about how to design a Bigtable schema for - # the best performance, see the documentation: - # - # https://cloud.google.com/bigtable/docs/schema-design - row_key = "greeting{}".format(i).encode() - row_mutation = RowMutationEntry( - row_key, SetCell(column_family_id, column, value) - ) - mutations.append(row_mutation) - await table.bulk_mutate_rows(mutations) - # [END bigtable_async_hw_write_rows] - - # [START bigtable_async_hw_create_filter] - # Create a filter to only retrieve the most recent version of the cell - # for each column across entire row. - row_filter = row_filters.CellsColumnLimitFilter(1) - # [END bigtable_async_hw_create_filter] - - # [START bigtable_async_hw_get_with_filter] - # [START bigtable_async_hw_get_by_key] - print("Getting a single greeting by row key.") - key = "greeting0".encode() - - row = await table.read_row(key, row_filter=row_filter) - cell = row.cells[0] - print(cell.value.decode("utf-8")) - # [END bigtable_async_hw_get_by_key] - # [END bigtable_async_hw_get_with_filter] - - # [START bigtable_async_hw_scan_with_filter] - # [START bigtable_async_hw_scan_all] - print("Scanning for all greetings:") - query = ReadRowsQuery(row_filter=row_filter) - async for row in await table.read_rows_stream(query): + try: + # let table creation complete + wait_for_table(admin_table) + # [START bigtable_async_hw_write_rows] + print("Writing some greetings to the table.") + greetings = ["Hello World!", "Hello Cloud Bigtable!", "Hello Python!"] + mutations = [] + column = "greeting" + for i, value in enumerate(greetings): + # Note: This example uses sequential numeric IDs for simplicity, + # but this can result in poor performance in a production + # application. Since rows are stored in sorted order by key, + # sequential keys can result in poor distribution of operations + # across nodes. + # + # For more information about how to design a Bigtable schema for + # the best performance, see the documentation: + # + # https://cloud.google.com/bigtable/docs/schema-design + row_key = "greeting{}".format(i).encode() + row_mutation = RowMutationEntry( + row_key, SetCell(column_family_id, column, value) + ) + mutations.append(row_mutation) + await table.bulk_mutate_rows(mutations) + # [END bigtable_async_hw_write_rows] + + # [START bigtable_async_hw_create_filter] + # Create a filter to only retrieve the most recent version of the cell + # for each column across entire row. + row_filter = row_filters.CellsColumnLimitFilter(1) + # [END bigtable_async_hw_create_filter] + + # [START bigtable_async_hw_get_with_filter] + # [START bigtable_async_hw_get_by_key] + print("Getting a single greeting by row key.") + key = "greeting0".encode() + + row = await table.read_row(key, row_filter=row_filter) cell = row.cells[0] print(cell.value.decode("utf-8")) - # [END bigtable_async_hw_scan_all] - # [END bigtable_async_hw_scan_with_filter] - - # [START bigtable_async_hw_delete_table] - # the async client only supports the data API. Table deletion as an admin operation - # use admin client to create the table - print("Deleting the {} table.".format(table_id)) - admin_table.delete() - # [END bigtable_async_hw_delete_table] + # [END bigtable_async_hw_get_by_key] + # [END bigtable_async_hw_get_with_filter] + + # [START bigtable_async_hw_scan_with_filter] + # [START bigtable_async_hw_scan_all] + print("Scanning for all greetings:") + query = ReadRowsQuery(row_filter=row_filter) + async for row in await table.read_rows_stream(query): + cell = row.cells[0] + print(cell.value.decode("utf-8")) + # [END bigtable_async_hw_scan_all] + # [END bigtable_async_hw_scan_with_filter] + finally: + # [START bigtable_async_hw_delete_table] + # the async client only supports the data API. Table deletion as an admin operation + # use admin client to create the table + print("Deleting the {} table.".format(table_id)) + admin_table.delete() + # [END bigtable_async_hw_delete_table] if __name__ == "__main__": diff --git a/samples/hello/async_main_test.py b/samples/hello/async_main_test.py index a47ac2d33..aa65a8652 100644 --- a/samples/hello/async_main_test.py +++ b/samples/hello/async_main_test.py @@ -13,27 +13,24 @@ # limitations under the License. import os -import random import asyncio +import uuid -from async_main import main +from .async_main import main PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] BIGTABLE_INSTANCE = os.environ["BIGTABLE_INSTANCE"] -TABLE_NAME_FORMAT = "hello-world-test-{}" -TABLE_NAME_RANGE = 10000 +TABLE_ID = f"hello-world-test-async-{str(uuid.uuid4())[:16]}" def test_async_main(capsys): - table_name = TABLE_NAME_FORMAT.format(random.randrange(TABLE_NAME_RANGE)) - - asyncio.run(main(PROJECT, BIGTABLE_INSTANCE, table_name)) + asyncio.run(main(PROJECT, BIGTABLE_INSTANCE, TABLE_ID)) out, _ = capsys.readouterr() - assert "Creating the {} table.".format(table_name) in out + assert "Creating the {} table.".format(TABLE_ID) in out assert "Writing some greetings to the table." in out assert "Getting a single greeting by row key." in out assert "Hello World!" in out assert "Scanning for all greetings" in out assert "Hello Cloud Bigtable!" in out - assert "Deleting the {} table.".format(table_name) in out + assert "Deleting the {} table.".format(TABLE_ID) in out diff --git a/samples/hello/main.py b/samples/hello/main.py index 3b7de34b0..3e5078608 100644 --- a/samples/hello/main.py +++ b/samples/hello/main.py @@ -25,6 +25,7 @@ """ import argparse +from ..utils import wait_for_table # [START bigtable_hw_imports] import datetime @@ -60,63 +61,68 @@ def main(project_id, instance_id, table_id): print("Table {} already exists.".format(table_id)) # [END bigtable_hw_create_table] - # [START bigtable_hw_write_rows] - print("Writing some greetings to the table.") - greetings = ["Hello World!", "Hello Cloud Bigtable!", "Hello Python!"] - rows = [] - column = "greeting".encode() - for i, value in enumerate(greetings): - # Note: This example uses sequential numeric IDs for simplicity, - # but this can result in poor performance in a production - # application. Since rows are stored in sorted order by key, - # sequential keys can result in poor distribution of operations - # across nodes. - # - # For more information about how to design a Bigtable schema for - # the best performance, see the documentation: - # - # https://cloud.google.com/bigtable/docs/schema-design - row_key = "greeting{}".format(i).encode() - row = table.direct_row(row_key) - row.set_cell( - column_family_id, column, value, timestamp=datetime.datetime.utcnow() - ) - rows.append(row) - table.mutate_rows(rows) - # [END bigtable_hw_write_rows] - - # [START bigtable_hw_create_filter] - # Create a filter to only retrieve the most recent version of the cell - # for each column across entire row. - row_filter = row_filters.CellsColumnLimitFilter(1) - # [END bigtable_hw_create_filter] - - # [START bigtable_hw_get_with_filter] - # [START bigtable_hw_get_by_key] - print("Getting a single greeting by row key.") - key = "greeting0".encode() - - row = table.read_row(key, row_filter) - cell = row.cells[column_family_id][column][0] - print(cell.value.decode("utf-8")) - # [END bigtable_hw_get_by_key] - # [END bigtable_hw_get_with_filter] - - # [START bigtable_hw_scan_with_filter] - # [START bigtable_hw_scan_all] - print("Scanning for all greetings:") - partial_rows = table.read_rows(filter_=row_filter) - - for row in partial_rows: + try: + # let table creation complete + wait_for_table(table) + + # [START bigtable_hw_write_rows] + print("Writing some greetings to the table.") + greetings = ["Hello World!", "Hello Cloud Bigtable!", "Hello Python!"] + rows = [] + column = "greeting".encode() + for i, value in enumerate(greetings): + # Note: This example uses sequential numeric IDs for simplicity, + # but this can result in poor performance in a production + # application. Since rows are stored in sorted order by key, + # sequential keys can result in poor distribution of operations + # across nodes. + # + # For more information about how to design a Bigtable schema for + # the best performance, see the documentation: + # + # https://cloud.google.com/bigtable/docs/schema-design + row_key = "greeting{}".format(i).encode() + row = table.direct_row(row_key) + row.set_cell( + column_family_id, column, value, timestamp=datetime.datetime.utcnow() + ) + rows.append(row) + table.mutate_rows(rows) + # [END bigtable_hw_write_rows] + + # [START bigtable_hw_create_filter] + # Create a filter to only retrieve the most recent version of the cell + # for each column across entire row. + row_filter = row_filters.CellsColumnLimitFilter(1) + # [END bigtable_hw_create_filter] + + # [START bigtable_hw_get_with_filter] + # [START bigtable_hw_get_by_key] + print("Getting a single greeting by row key.") + key = "greeting0".encode() + + row = table.read_row(key, row_filter) cell = row.cells[column_family_id][column][0] print(cell.value.decode("utf-8")) - # [END bigtable_hw_scan_all] - # [END bigtable_hw_scan_with_filter] - - # [START bigtable_hw_delete_table] - print("Deleting the {} table.".format(table_id)) - table.delete() - # [END bigtable_hw_delete_table] + # [END bigtable_hw_get_by_key] + # [END bigtable_hw_get_with_filter] + + # [START bigtable_hw_scan_with_filter] + # [START bigtable_hw_scan_all] + print("Scanning for all greetings:") + partial_rows = table.read_rows(filter_=row_filter) + + for row in partial_rows: + cell = row.cells[column_family_id][column][0] + print(cell.value.decode("utf-8")) + # [END bigtable_hw_scan_all] + # [END bigtable_hw_scan_with_filter] + + finally: + # [START bigtable_hw_delete_table] + print("Deleting the {} table.".format(table_id)) + table.delete() + # [END bigtable_hw_delete_table] if __name__ == "__main__": diff --git a/samples/hello/main_test.py b/samples/hello/main_test.py index 641b34d11..28814d909 100644 --- a/samples/hello/main_test.py +++ b/samples/hello/main_test.py @@ -13,26 +13,23 @@ # limitations under the License. import os -import random +import uuid -from main import main +from .main import main PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] BIGTABLE_INSTANCE = os.environ["BIGTABLE_INSTANCE"] -TABLE_NAME_FORMAT = "hello-world-test-{}" -TABLE_NAME_RANGE = 10000 +TABLE_ID = f"hello-world-test-{str(uuid.uuid4())[:16]}" def test_main(capsys): - table_name = TABLE_NAME_FORMAT.format(random.randrange(TABLE_NAME_RANGE)) - - main(PROJECT, BIGTABLE_INSTANCE, table_name) + main(PROJECT, BIGTABLE_INSTANCE, TABLE_ID) out, _ = capsys.readouterr() - assert "Creating the {} table.".format(table_name) in out + assert "Creating the {} table.".format(TABLE_ID) in out assert "Writing some greetings to the table." in out assert "Getting a single greeting by row key." in out assert "Hello World!" in out assert "Scanning for all greetings" in out assert "Hello Cloud Bigtable!" in out - assert "Deleting the {} table.".format(table_name) in out + assert "Deleting the {} table.".format(TABLE_ID) in out diff --git a/samples/hello_happybase/__init__.py b/samples/hello_happybase/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/samples/hello_happybase/main.py b/samples/hello_happybase/main.py index 7999fd006..50820febd 100644 --- a/samples/hello_happybase/main.py +++ b/samples/hello_happybase/main.py @@ -25,6 +25,7 @@ """ import argparse +from ..utils import wait_for_table # [START bigtable_hw_imports_happybase] from google.cloud import bigtable @@ -51,6 +52,8 @@ def main(project_id, instance_id, table_name): ) # [END bigtable_hw_create_table_happybase] + wait_for_table(instance.table(table_name)) + # [START bigtable_hw_write_rows_happybase] print("Writing some greetings to the table.") table = connection.table(table_name) @@ -90,12 +93,11 @@ def main(project_id, instance_id, table_name): print("\t{}: {}".format(key, row[column_name.encode("utf-8")])) # [END bigtable_hw_scan_all_happybase] + finally: # [START bigtable_hw_delete_table_happybase] print("Deleting the {} table.".format(table_name)) connection.delete_table(table_name) # [END bigtable_hw_delete_table_happybase] - - finally: connection.close() diff --git a/samples/hello_happybase/main_test.py b/samples/hello_happybase/main_test.py index 6a63750da..252f4ccaf 100644 --- a/samples/hello_happybase/main_test.py +++ b/samples/hello_happybase/main_test.py @@ -13,25 +13,32 @@ # limitations under the License. import os -import random +import uuid -from main import main +from .main import main +from google.cloud import bigtable PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] BIGTABLE_INSTANCE = os.environ["BIGTABLE_INSTANCE"] -TABLE_NAME_FORMAT = "hello-world-hb-test-{}" -TABLE_NAME_RANGE = 10000 +TABLE_ID = f"hello-world-hb-test-{str(uuid.uuid4())[:16]}" def test_main(capsys): - table_name = TABLE_NAME_FORMAT.format(random.randrange(TABLE_NAME_RANGE)) - main(PROJECT, BIGTABLE_INSTANCE, table_name) + try: + main(PROJECT, BIGTABLE_INSTANCE, TABLE_ID) - out, _ = capsys.readouterr() - assert "Creating the {} table.".format(table_name) in out - assert "Writing some greetings to the table." in out - assert "Getting a single greeting by row key." in out - assert "Hello World!" in out - assert "Scanning for all greetings" in out - assert "Hello Cloud Bigtable!" in out - assert "Deleting the {} table.".format(table_name) in out + out, _ = capsys.readouterr() + assert "Creating the {} table.".format(TABLE_ID) in out + assert "Writing some greetings to the table." in out + assert "Getting a single greeting by row key." in out + assert "Hello World!" in out + assert "Scanning for all greetings" in out + assert "Hello Cloud Bigtable!" in out + assert "Deleting the {} table.".format(TABLE_ID) in out + finally: + # delete table + client = bigtable.Client(PROJECT, admin=True) + instance = client.instance(BIGTABLE_INSTANCE) + table = instance.table(TABLE_ID) + if table.exists(): + table.delete() diff --git a/samples/quickstart/__init__.py b/samples/quickstart/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/samples/quickstart/main_async_test.py b/samples/quickstart/main_async_test.py index 841cfc180..0749cbd31 100644 --- a/samples/quickstart/main_async_test.py +++ b/samples/quickstart/main_async_test.py @@ -13,46 +13,26 @@ # limitations under the License. import os +import uuid from typing import AsyncGenerator from google.cloud.bigtable.data import BigtableDataClientAsync, SetCell import pytest import pytest_asyncio -from main_async import main - +from .main_async import main +from ..utils import create_table_cm PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] BIGTABLE_INSTANCE = os.environ["BIGTABLE_INSTANCE"] -TABLE_ID_FORMAT = "quickstart-test-{}" +TABLE_ID = f"quickstart-async-test-{str(uuid.uuid4())[:16]}" @pytest_asyncio.fixture async def table_id() -> AsyncGenerator[str, None]: - table_id = _create_table() - await _populate_table(table_id) - - yield table_id - - _delete_table(table_id) - - -def _create_table(): - from google.cloud import bigtable - import uuid - - client = bigtable.Client(project=PROJECT, admin=True) - instance = client.instance(BIGTABLE_INSTANCE) - - table_id = TABLE_ID_FORMAT.format(uuid.uuid4().hex[:8]) - table = instance.table(table_id) - if table.exists(): - table.delete() - - table.create(column_families={"cf1": None}) - - client.close() - return table_id + with create_table_cm(PROJECT, BIGTABLE_INSTANCE, TABLE_ID, {"cf1": None}): + await _populate_table(TABLE_ID) + yield TABLE_ID async def _populate_table(table_id: str): @@ -61,16 +41,6 @@ async def _populate_table(table_id: str): await table.mutate_row("r1", SetCell("cf1", "c1", "test-value")) -def _delete_table(table_id: str): - from google.cloud import bigtable - - client = bigtable.Client(project=PROJECT, admin=True) - instance = client.instance(BIGTABLE_INSTANCE) - table = instance.table(table_id) - table.delete() - client.close() - - @pytest.mark.asyncio async def test_main(capsys, table_id): await main(PROJECT, BIGTABLE_INSTANCE, table_id) diff --git a/samples/quickstart/main_test.py b/samples/quickstart/main_test.py index 46d578b6b..f58161f23 100644 --- a/samples/quickstart/main_test.py +++ b/samples/quickstart/main_test.py @@ -14,35 +14,28 @@ import os import uuid - -from google.cloud import bigtable import pytest -from main import main +from .main import main + +from ..utils import create_table_cm PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] BIGTABLE_INSTANCE = os.environ["BIGTABLE_INSTANCE"] -TABLE_ID_FORMAT = "quickstart-test-{}" +TABLE_ID = f"quickstart-test-{str(uuid.uuid4())[:16]}" @pytest.fixture() def table(): - table_id = TABLE_ID_FORMAT.format(uuid.uuid4().hex[:8]) - client = bigtable.Client(project=PROJECT, admin=True) - instance = client.instance(BIGTABLE_INSTANCE) - table = instance.table(table_id) column_family_id = "cf1" column_families = {column_family_id: None} - table.create(column_families=column_families) - - row = table.direct_row("r1") - row.set_cell(column_family_id, "c1", "test-value") - row.commit() - - yield table_id + with create_table_cm(PROJECT, BIGTABLE_INSTANCE, TABLE_ID, column_families) as table: + row = table.direct_row("r1") + row.set_cell(column_family_id, "c1", "test-value") + row.commit() - table.delete() + yield TABLE_ID def test_main(capsys, table): diff --git a/samples/quickstart_happybase/__init__.py b/samples/quickstart_happybase/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/samples/quickstart_happybase/main_test.py b/samples/quickstart_happybase/main_test.py index dc62ebede..343ec800a 100644 --- a/samples/quickstart_happybase/main_test.py +++ b/samples/quickstart_happybase/main_test.py @@ -14,35 +14,26 @@ import os import uuid - -from google.cloud import bigtable import pytest -from main import main - +from .main import main +from ..utils import create_table_cm PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] BIGTABLE_INSTANCE = os.environ["BIGTABLE_INSTANCE"] -TABLE_ID_FORMAT = "quickstart-hb-test-{}" +TABLE_ID = f"quickstart-hb-test-{str(uuid.uuid4())[:16]}" @pytest.fixture() def table(): - table_id = TABLE_ID_FORMAT.format(uuid.uuid4().hex[:8]) - client = bigtable.Client(project=PROJECT, admin=True) - instance = client.instance(BIGTABLE_INSTANCE) - table = instance.table(table_id) column_family_id = "cf1" column_families = {column_family_id: None} - table.create(column_families=column_families) - - row = table.direct_row("r1") - row.set_cell(column_family_id, "c1", "test-value") - row.commit() - - yield table_id + with create_table_cm(PROJECT, BIGTABLE_INSTANCE, TABLE_ID, column_families) as table: + row = table.direct_row("r1") + row.set_cell(column_family_id, "c1", "test-value") + row.commit() - table.delete() + yield TABLE_ID def test_main(capsys, table): diff --git a/samples/snippets/__init__.py b/samples/snippets/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/samples/snippets/data_client/__init__.py b/samples/snippets/data_client/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/samples/snippets/data_client/data_client_snippets_async_test.py b/samples/snippets/data_client/data_client_snippets_async_test.py index 2e0fb9b81..8dfff50d1 100644 --- a/samples/snippets/data_client/data_client_snippets_async_test.py +++ b/samples/snippets/data_client/data_client_snippets_async_test.py @@ -12,36 +12,22 @@ # limitations under the License. import pytest import pytest_asyncio -import uuid import os +import uuid -import data_client_snippets_async as data_snippets +from . import data_client_snippets_async as data_snippets +from ...utils import create_table_cm PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] BIGTABLE_INSTANCE = os.environ["BIGTABLE_INSTANCE"] -TABLE_ID_STATIC = os.getenv( - "BIGTABLE_TABLE", None -) # if not set, a temproary table will be generated +TABLE_ID = f"data-client-{str(uuid.uuid4())[:16]}" @pytest.fixture(scope="session") def table_id(): - from google.cloud import bigtable - - client = bigtable.Client(project=PROJECT, admin=True) - instance = client.instance(BIGTABLE_INSTANCE) - table_id = TABLE_ID_STATIC or f"data-client-{str(uuid.uuid4())[:16]}" - - admin_table = instance.table(table_id) - if not admin_table.exists(): - admin_table.create(column_families={"family": None, "stats_summary": None}) - - yield table_id - - if not table_id == TABLE_ID_STATIC: - # clean up table when finished - admin_table.delete() + with create_table_cm(PROJECT, BIGTABLE_INSTANCE, TABLE_ID, {"family": None, "stats_summary": None}): + yield TABLE_ID @pytest_asyncio.fixture diff --git a/samples/snippets/deletes/__init__.py b/samples/snippets/deletes/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/samples/snippets/deletes/deletes_async_test.py b/samples/snippets/deletes/deletes_async_test.py index b708bd52e..9408a8320 100644 --- a/samples/snippets/deletes/deletes_async_test.py +++ b/samples/snippets/deletes/deletes_async_test.py @@ -15,52 +15,26 @@ import datetime import os +import uuid from typing import AsyncGenerator from google.cloud._helpers import _microseconds_from_datetime import pytest import pytest_asyncio -import deletes_snippets_async +from . import deletes_snippets_async +from ...utils import create_table_cm PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] BIGTABLE_INSTANCE = os.environ["BIGTABLE_INSTANCE"] -TABLE_ID_PREFIX = "mobile-time-series-{}" +TABLE_ID = f"mobile-time-series-deletes-async-{str(uuid.uuid4())[:16]}" -@pytest_asyncio.fixture +@pytest_asyncio.fixture(scope="module", autouse=True) async def table_id() -> AsyncGenerator[str, None]: - table_id = _create_table() - await _populate_table(table_id) - yield table_id - _delete_table(table_id) - - -def _create_table(): - from google.cloud import bigtable - import uuid - - client = bigtable.Client(project=PROJECT, admin=True) - instance = client.instance(BIGTABLE_INSTANCE) - - table_id = TABLE_ID_PREFIX.format(str(uuid.uuid4())[:16]) - table = instance.table(table_id) - if table.exists(): - table.delete() - - table.create(column_families={"stats_summary": None, "cell_plan": None}) - client.close() - return table_id - - -def _delete_table(table_id: str): - from google.cloud import bigtable - - client = bigtable.Client(project=PROJECT, admin=True) - instance = client.instance(BIGTABLE_INSTANCE) - table = instance.table(table_id) - table.delete() - client.close() + with create_table_cm(PROJECT, BIGTABLE_INSTANCE, TABLE_ID, {"stats_summary": None, "cell_plan": None}, verbose=False): + await _populate_table(TABLE_ID) + yield TABLE_ID async def _populate_table(table_id): diff --git a/samples/snippets/deletes/deletes_test.py b/samples/snippets/deletes/deletes_test.py index bebaabafb..3284c37da 100644 --- a/samples/snippets/deletes/deletes_test.py +++ b/samples/snippets/deletes/deletes_test.py @@ -18,81 +18,72 @@ import time import uuid -from google.cloud import bigtable import pytest -import deletes_snippets +from . import deletes_snippets +from ...utils import create_table_cm PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] BIGTABLE_INSTANCE = os.environ["BIGTABLE_INSTANCE"] -TABLE_ID_PREFIX = "mobile-time-series-{}" +TABLE_ID = f"mobile-time-series-deletes-{str(uuid.uuid4())[:16]}" -@pytest.fixture(scope="module", autouse=True) +@pytest.fixture(scope="module") def table_id(): from google.cloud.bigtable.row_set import RowSet - client = bigtable.Client(project=PROJECT, admin=True) - instance = client.instance(BIGTABLE_INSTANCE) - - table_id = TABLE_ID_PREFIX.format(str(uuid.uuid4())[:16]) - table = instance.table(table_id) - if table.exists(): - table.delete() - - table.create(column_families={"stats_summary": None, "cell_plan": None}) - - timestamp = datetime.datetime(2019, 5, 1) - timestamp_minus_hr = datetime.datetime(2019, 5, 1) - datetime.timedelta(hours=1) - - row_keys = [ - "phone#4c410523#20190501", - "phone#4c410523#20190502", - "phone#4c410523#20190505", - "phone#5c10102#20190501", - "phone#5c10102#20190502", - ] - - rows = [table.direct_row(row_key) for row_key in row_keys] - - rows[0].set_cell("stats_summary", "connected_cell", 1, timestamp) - rows[0].set_cell("stats_summary", "connected_wifi", 1, timestamp) - rows[0].set_cell("stats_summary", "os_build", "PQ2A.190405.003", timestamp) - rows[0].set_cell("cell_plan", "data_plan_01gb", "true", timestamp_minus_hr) - rows[0].set_cell("cell_plan", "data_plan_01gb", "false", timestamp) - rows[0].set_cell("cell_plan", "data_plan_05gb", "true", timestamp) - rows[1].set_cell("stats_summary", "connected_cell", 1, timestamp) - rows[1].set_cell("stats_summary", "connected_wifi", 1, timestamp) - rows[1].set_cell("stats_summary", "os_build", "PQ2A.190405.004", timestamp) - rows[1].set_cell("cell_plan", "data_plan_05gb", "true", timestamp) - rows[2].set_cell("stats_summary", "connected_cell", 0, timestamp) - rows[2].set_cell("stats_summary", "connected_wifi", 1, timestamp) - rows[2].set_cell("stats_summary", "os_build", "PQ2A.190406.000", timestamp) - rows[2].set_cell("cell_plan", "data_plan_05gb", "true", timestamp) - rows[3].set_cell("stats_summary", "connected_cell", 1, timestamp) - rows[3].set_cell("stats_summary", "connected_wifi", 1, timestamp) - rows[3].set_cell("stats_summary", "os_build", "PQ2A.190401.002", timestamp) - rows[3].set_cell("cell_plan", "data_plan_10gb", "true", timestamp) - rows[4].set_cell("stats_summary", "connected_cell", 1, timestamp) - rows[4].set_cell("stats_summary", "connected_wifi", 0, timestamp) - rows[4].set_cell("stats_summary", "os_build", "PQ2A.190406.000", timestamp) - rows[4].set_cell("cell_plan", "data_plan_10gb", "true", timestamp) - - table.mutate_rows(rows) - - # Ensure mutations have propagated. - row_set = RowSet() - - for row_key in row_keys: - row_set.add_row_key(row_key) - - fetched = list(table.read_rows(row_set=row_set)) - - while len(fetched) < len(rows): - time.sleep(5) + with create_table_cm(PROJECT, BIGTABLE_INSTANCE, TABLE_ID, {"stats_summary": None, "cell_plan": None}, verbose=False) as table: + timestamp = datetime.datetime(2019, 5, 1) + timestamp_minus_hr = datetime.datetime(2019, 5, 1) - datetime.timedelta(hours=1) + + row_keys = [ + "phone#4c410523#20190501", + "phone#4c410523#20190502", + "phone#4c410523#20190505", + "phone#5c10102#20190501", + "phone#5c10102#20190502", + ] + + rows = [table.direct_row(row_key) for row_key in row_keys] + + rows[0].set_cell("stats_summary", "connected_cell", 1, timestamp) + rows[0].set_cell("stats_summary", "connected_wifi", 1, timestamp) + rows[0].set_cell("stats_summary", "os_build", "PQ2A.190405.003", timestamp) + rows[0].set_cell("cell_plan", "data_plan_01gb", "true", timestamp_minus_hr) + rows[0].set_cell("cell_plan", "data_plan_01gb", "false", timestamp) + rows[0].set_cell("cell_plan", "data_plan_05gb", "true", timestamp) + rows[1].set_cell("stats_summary", "connected_cell", 1, timestamp) + rows[1].set_cell("stats_summary", "connected_wifi", 1, timestamp) + rows[1].set_cell("stats_summary", "os_build", "PQ2A.190405.004", timestamp) + rows[1].set_cell("cell_plan", "data_plan_05gb", "true", timestamp) + rows[2].set_cell("stats_summary", "connected_cell", 0, timestamp) + rows[2].set_cell("stats_summary", "connected_wifi", 1, timestamp) + rows[2].set_cell("stats_summary", "os_build", "PQ2A.190406.000", timestamp) + rows[2].set_cell("cell_plan", "data_plan_05gb", "true", timestamp) + rows[3].set_cell("stats_summary", "connected_cell", 1, timestamp) + rows[3].set_cell("stats_summary", "connected_wifi", 1, timestamp) + rows[3].set_cell("stats_summary", "os_build", "PQ2A.190401.002", timestamp) + rows[3].set_cell("cell_plan", "data_plan_10gb", "true", timestamp) + rows[4].set_cell("stats_summary", "connected_cell", 1, timestamp) + rows[4].set_cell("stats_summary", "connected_wifi", 0, timestamp) + rows[4].set_cell("stats_summary", "os_build", "PQ2A.190406.000", timestamp) + rows[4].set_cell("cell_plan", "data_plan_10gb", "true", timestamp) + + table.mutate_rows(rows) + + # Ensure mutations have propagated. + row_set = RowSet() + + for row_key in row_keys: + row_set.add_row_key(row_key) + fetched = list(table.read_rows(row_set=row_set)) - yield table_id + while len(fetched) < len(rows): + time.sleep(5) + fetched = list(table.read_rows(row_set=row_set)) + + yield TABLE_ID def assert_output_match(capsys, expected): @@ -135,6 +126,8 @@ def test_delete_column_family(capsys, table_id): assert_output_match(capsys, "") -def test_delete_table(capsys, table_id): - deletes_snippets.delete_table(PROJECT, BIGTABLE_INSTANCE, table_id) - assert_output_match(capsys, "") +def test_delete_table(capsys): + delete_table_id = f"to-delete-table-{str(uuid.uuid4())[:16]}" + with create_table_cm(PROJECT, BIGTABLE_INSTANCE, delete_table_id, verbose=False): + deletes_snippets.delete_table(PROJECT, BIGTABLE_INSTANCE, delete_table_id) + assert_output_match(capsys, "") diff --git a/samples/snippets/filters/filter_snippets_async_test.py b/samples/snippets/filters/filter_snippets_async_test.py index 76751feaf..124db8157 100644 --- a/samples/snippets/filters/filter_snippets_async_test.py +++ b/samples/snippets/filters/filter_snippets_async_test.py @@ -14,6 +14,7 @@ import datetime import os +import uuid import inspect from typing import AsyncGenerator @@ -23,46 +24,21 @@ from .snapshots.snap_filters_test import snapshots from . import filter_snippets_async +from ...utils import create_table_cm from google.cloud._helpers import ( _microseconds_from_datetime, ) PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] BIGTABLE_INSTANCE = os.environ["BIGTABLE_INSTANCE"] -TABLE_ID_PREFIX = "mobile-time-series-{}" +TABLE_ID = f"mobile-time-series-filters-async-{str(uuid.uuid4())[:16]}" -@pytest_asyncio.fixture +@pytest_asyncio.fixture(scope="module", autouse=True) async def table_id() -> AsyncGenerator[str, None]: - table_id = _create_table() - await _populate_table(table_id) - yield table_id - _delete_table(table_id) - - -def _create_table(): - from google.cloud import bigtable - import uuid - - client = bigtable.Client(project=PROJECT, admin=True) - instance = client.instance(BIGTABLE_INSTANCE) - - table_id = TABLE_ID_PREFIX.format(str(uuid.uuid4())[:16]) - table = instance.table(table_id) - if table.exists(): - table.delete() - - table.create(column_families={"stats_summary": None, "cell_plan": None}) - return table_id - - -def _delete_table(table_id: str): - from google.cloud import bigtable - - client = bigtable.Client(project=PROJECT, admin=True) - instance = client.instance(BIGTABLE_INSTANCE) - table = instance.table(table_id) - table.delete() + with create_table_cm(PROJECT, BIGTABLE_INSTANCE, TABLE_ID, {"stats_summary": None, "cell_plan": None}): + await _populate_table(TABLE_ID) + yield TABLE_ID async def _populate_table(table_id): diff --git a/samples/snippets/filters/filters_test.py b/samples/snippets/filters/filters_test.py index a84932039..fe99886bd 100644 --- a/samples/snippets/filters/filters_test.py +++ b/samples/snippets/filters/filters_test.py @@ -18,84 +18,75 @@ import time import uuid -from google.cloud import bigtable import pytest from . import filter_snippets from .snapshots.snap_filters_test import snapshots +from ...utils import create_table_cm PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] BIGTABLE_INSTANCE = os.environ["BIGTABLE_INSTANCE"] -TABLE_ID_PREFIX = "mobile-time-series-{}" +TABLE_ID = f"mobile-time-series-filters-{str(uuid.uuid4())[:16]}" @pytest.fixture(scope="module", autouse=True) def table_id(): from google.cloud.bigtable.row_set import RowSet - client = bigtable.Client(project=PROJECT, admin=True) - instance = client.instance(BIGTABLE_INSTANCE) - - table_id = TABLE_ID_PREFIX.format(str(uuid.uuid4())[:16]) - table = instance.table(table_id) - if table.exists(): - table.delete() - - table.create(column_families={"stats_summary": None, "cell_plan": None}) - - timestamp = datetime.datetime(2019, 5, 1) - timestamp_minus_hr = datetime.datetime(2019, 5, 1) - datetime.timedelta(hours=1) - - row_keys = [ - "phone#4c410523#20190501", - "phone#4c410523#20190502", - "phone#4c410523#20190505", - "phone#5c10102#20190501", - "phone#5c10102#20190502", - ] - - rows = [table.direct_row(row_key) for row_key in row_keys] - - rows[0].set_cell("stats_summary", "connected_cell", 1, timestamp) - rows[0].set_cell("stats_summary", "connected_wifi", 1, timestamp) - rows[0].set_cell("stats_summary", "os_build", "PQ2A.190405.003", timestamp) - rows[0].set_cell("cell_plan", "data_plan_01gb", "true", timestamp_minus_hr) - rows[0].set_cell("cell_plan", "data_plan_01gb", "false", timestamp) - rows[0].set_cell("cell_plan", "data_plan_05gb", "true", timestamp) - rows[1].set_cell("stats_summary", "connected_cell", 1, timestamp) - rows[1].set_cell("stats_summary", "connected_wifi", 1, timestamp) - rows[1].set_cell("stats_summary", "os_build", "PQ2A.190405.004", timestamp) - rows[1].set_cell("cell_plan", "data_plan_05gb", "true", timestamp) - rows[2].set_cell("stats_summary", "connected_cell", 0, timestamp) - rows[2].set_cell("stats_summary", "connected_wifi", 1, timestamp) - rows[2].set_cell("stats_summary", "os_build", "PQ2A.190406.000", timestamp) - rows[2].set_cell("cell_plan", "data_plan_05gb", "true", timestamp) - rows[3].set_cell("stats_summary", "connected_cell", 1, timestamp) - rows[3].set_cell("stats_summary", "connected_wifi", 1, timestamp) - rows[3].set_cell("stats_summary", "os_build", "PQ2A.190401.002", timestamp) - rows[3].set_cell("cell_plan", "data_plan_10gb", "true", timestamp) - rows[4].set_cell("stats_summary", "connected_cell", 1, timestamp) - rows[4].set_cell("stats_summary", "connected_wifi", 0, timestamp) - rows[4].set_cell("stats_summary", "os_build", "PQ2A.190406.000", timestamp) - rows[4].set_cell("cell_plan", "data_plan_10gb", "true", timestamp) - - table.mutate_rows(rows) - - # Ensure mutations have propagated. - row_set = RowSet() - - for row_key in row_keys: - row_set.add_row_key(row_key) - - fetched = list(table.read_rows(row_set=row_set)) - - while len(fetched) < len(rows): - time.sleep(5) + table_id = TABLE_ID + with create_table_cm(PROJECT, BIGTABLE_INSTANCE, table_id, {"stats_summary": None, "cell_plan": None}) as table: + + timestamp = datetime.datetime(2019, 5, 1) + timestamp_minus_hr = datetime.datetime(2019, 5, 1) - datetime.timedelta(hours=1) + + row_keys = [ + "phone#4c410523#20190501", + "phone#4c410523#20190502", + "phone#4c410523#20190505", + "phone#5c10102#20190501", + "phone#5c10102#20190502", + ] + + rows = [table.direct_row(row_key) for row_key in row_keys] + + rows[0].set_cell("stats_summary", "connected_cell", 1, timestamp) + rows[0].set_cell("stats_summary", "connected_wifi", 1, timestamp) + rows[0].set_cell("stats_summary", "os_build", "PQ2A.190405.003", timestamp) + rows[0].set_cell("cell_plan", "data_plan_01gb", "true", timestamp_minus_hr) + rows[0].set_cell("cell_plan", "data_plan_01gb", "false", timestamp) + rows[0].set_cell("cell_plan", "data_plan_05gb", "true", timestamp) + rows[1].set_cell("stats_summary", "connected_cell", 1, timestamp) + rows[1].set_cell("stats_summary", "connected_wifi", 1, timestamp) + rows[1].set_cell("stats_summary", "os_build", "PQ2A.190405.004", timestamp) + rows[1].set_cell("cell_plan", "data_plan_05gb", "true", timestamp) + rows[2].set_cell("stats_summary", "connected_cell", 0, timestamp) + rows[2].set_cell("stats_summary", "connected_wifi", 1, timestamp) + rows[2].set_cell("stats_summary", "os_build", "PQ2A.190406.000", timestamp) + rows[2].set_cell("cell_plan", "data_plan_05gb", "true", timestamp) + rows[3].set_cell("stats_summary", "connected_cell", 1, timestamp) + rows[3].set_cell("stats_summary", "connected_wifi", 1, timestamp) + rows[3].set_cell("stats_summary", "os_build", "PQ2A.190401.002", timestamp) + rows[3].set_cell("cell_plan", "data_plan_10gb", "true", timestamp) + rows[4].set_cell("stats_summary", "connected_cell", 1, timestamp) + rows[4].set_cell("stats_summary", "connected_wifi", 0, timestamp) + rows[4].set_cell("stats_summary", "os_build", "PQ2A.190406.000", timestamp) + rows[4].set_cell("cell_plan", "data_plan_10gb", "true", timestamp) + + table.mutate_rows(rows) + + # Ensure mutations have propagated. + row_set = RowSet() + + for row_key in row_keys: + row_set.add_row_key(row_key) + fetched = list(table.read_rows(row_set=row_set)) - yield table_id + while len(fetched) < len(rows): + time.sleep(5) + fetched = list(table.read_rows(row_set=row_set)) - table.delete() + yield table_id def test_filter_limit_row_sample(capsys, table_id): diff --git a/samples/snippets/reads/reads_test.py b/samples/snippets/reads/reads_test.py index da826d6fb..0078ce598 100644 --- a/samples/snippets/reads/reads_test.py +++ b/samples/snippets/reads/reads_test.py @@ -13,65 +13,52 @@ import datetime import os -import uuid import inspect +import uuid -from google.cloud import bigtable import pytest from .snapshots.snap_reads_test import snapshots from . import read_snippets +from ...utils import create_table_cm PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] BIGTABLE_INSTANCE = os.environ["BIGTABLE_INSTANCE"] -TABLE_ID_PREFIX = "mobile-time-series-{}" +TABLE_ID = f"mobile-time-series-reads-{str(uuid.uuid4())[:16]}" @pytest.fixture(scope="module", autouse=True) def table_id(): - client = bigtable.Client(project=PROJECT, admin=True) - instance = client.instance(BIGTABLE_INSTANCE) - - table_id = TABLE_ID_PREFIX.format(str(uuid.uuid4())[:16]) - table = instance.table(table_id) - if table.exists(): - table.delete() - - table.create(column_families={"stats_summary": None}) - - # table = instance.table(table_id) - - timestamp = datetime.datetime(2019, 5, 1) - rows = [ - table.direct_row("phone#4c410523#20190501"), - table.direct_row("phone#4c410523#20190502"), - table.direct_row("phone#4c410523#20190505"), - table.direct_row("phone#5c10102#20190501"), - table.direct_row("phone#5c10102#20190502"), - ] - - rows[0].set_cell("stats_summary", "connected_cell", 1, timestamp) - rows[0].set_cell("stats_summary", "connected_wifi", 1, timestamp) - rows[0].set_cell("stats_summary", "os_build", "PQ2A.190405.003", timestamp) - rows[1].set_cell("stats_summary", "connected_cell", 1, timestamp) - rows[1].set_cell("stats_summary", "connected_wifi", 1, timestamp) - rows[1].set_cell("stats_summary", "os_build", "PQ2A.190405.004", timestamp) - rows[2].set_cell("stats_summary", "connected_cell", 0, timestamp) - rows[2].set_cell("stats_summary", "connected_wifi", 1, timestamp) - rows[2].set_cell("stats_summary", "os_build", "PQ2A.190406.000", timestamp) - rows[3].set_cell("stats_summary", "connected_cell", 1, timestamp) - rows[3].set_cell("stats_summary", "connected_wifi", 1, timestamp) - rows[3].set_cell("stats_summary", "os_build", "PQ2A.190401.002", timestamp) - rows[4].set_cell("stats_summary", "connected_cell", 1, timestamp) - rows[4].set_cell("stats_summary", "connected_wifi", 0, timestamp) - rows[4].set_cell("stats_summary", "os_build", "PQ2A.190406.000", timestamp) - - table.mutate_rows(rows) - - yield table_id - - table.delete() + with create_table_cm(PROJECT, BIGTABLE_INSTANCE, TABLE_ID, {"stats_summary": None}) as table: + timestamp = datetime.datetime(2019, 5, 1) + rows = [ + table.direct_row("phone#4c410523#20190501"), + table.direct_row("phone#4c410523#20190502"), + table.direct_row("phone#4c410523#20190505"), + table.direct_row("phone#5c10102#20190501"), + table.direct_row("phone#5c10102#20190502"), + ] + + rows[0].set_cell("stats_summary", "connected_cell", 1, timestamp) + rows[0].set_cell("stats_summary", "connected_wifi", 1, timestamp) + rows[0].set_cell("stats_summary", "os_build", "PQ2A.190405.003", timestamp) + rows[1].set_cell("stats_summary", "connected_cell", 1, timestamp) + rows[1].set_cell("stats_summary", "connected_wifi", 1, timestamp) + rows[1].set_cell("stats_summary", "os_build", "PQ2A.190405.004", timestamp) + rows[2].set_cell("stats_summary", "connected_cell", 0, timestamp) + rows[2].set_cell("stats_summary", "connected_wifi", 1, timestamp) + rows[2].set_cell("stats_summary", "os_build", "PQ2A.190406.000", timestamp) + rows[3].set_cell("stats_summary", "connected_cell", 1, timestamp) + rows[3].set_cell("stats_summary", "connected_wifi", 1, timestamp) + rows[3].set_cell("stats_summary", "os_build", "PQ2A.190401.002", timestamp) + rows[4].set_cell("stats_summary", "connected_cell", 1, timestamp) + rows[4].set_cell("stats_summary", "connected_wifi", 0, timestamp) + rows[4].set_cell("stats_summary", "os_build", "PQ2A.190406.000", timestamp) + + table.mutate_rows(rows) + + yield TABLE_ID def test_read_row(capsys, table_id): diff --git a/samples/snippets/writes/writes_test.py b/samples/snippets/writes/writes_test.py index 77ae883d6..2c7a3d62b 100644 --- a/samples/snippets/writes/writes_test.py +++ b/samples/snippets/writes/writes_test.py @@ -13,48 +13,27 @@ # limitations under the License. import os -import uuid import backoff from google.api_core.exceptions import DeadlineExceeded -from google.cloud import bigtable import pytest +import uuid from .write_batch import write_batch from .write_conditionally import write_conditional from .write_increment import write_increment from .write_simple import write_simple - +from ...utils import create_table_cm PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] BIGTABLE_INSTANCE = os.environ["BIGTABLE_INSTANCE"] -TABLE_ID_PREFIX = "mobile-time-series-{}" - - -@pytest.fixture -def bigtable_client(): - return bigtable.Client(project=PROJECT, admin=True) +TABLE_ID = f"mobile-time-series-writes-{str(uuid.uuid4())[:16]}" @pytest.fixture -def bigtable_instance(bigtable_client): - return bigtable_client.instance(BIGTABLE_INSTANCE) - - -@pytest.fixture -def table_id(bigtable_instance): - table_id = TABLE_ID_PREFIX.format(str(uuid.uuid4())[:16]) - table = bigtable_instance.table(table_id) - if table.exists(): - table.delete() - - column_family_id = "stats_summary" - column_families = {column_family_id: None} - table.create(column_families=column_families) - - yield table_id - - table.delete() +def table_id(): + with create_table_cm(PROJECT, BIGTABLE_INSTANCE, TABLE_ID, {"stats_summary": None}): + yield TABLE_ID def test_writes(capsys, table_id): diff --git a/samples/tableadmin/__init__.py b/samples/tableadmin/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/samples/tableadmin/tableadmin.py b/samples/tableadmin/tableadmin.py index 7c28601fb..ad00e5788 100644 --- a/samples/tableadmin/tableadmin.py +++ b/samples/tableadmin/tableadmin.py @@ -35,36 +35,7 @@ from google.cloud import bigtable from google.cloud.bigtable import column_family - - -def create_table(project_id, instance_id, table_id): - """Create a Bigtable table - - :type project_id: str - :param project_id: Project id of the client. - - :type instance_id: str - :param instance_id: Instance of the client. - - :type table_id: str - :param table_id: Table id to create table. - """ - - client = bigtable.Client(project=project_id, admin=True) - instance = client.instance(instance_id) - table = instance.table(table_id) - - # Check whether table exists in an instance. - # Create table if it does not exists. - print("Checking if table {} exists...".format(table_id)) - if table.exists(): - print("Table {} already exists.".format(table_id)) - else: - print("Creating the {} table.".format(table_id)) - table.create() - print("Created table {}.".format(table_id)) - - return client, instance, table +from ..utils import create_table_cm def run_table_operations(project_id, instance_id, table_id): @@ -80,154 +51,155 @@ def run_table_operations(project_id, instance_id, table_id): :param table_id: Table id to create table. """ - client, instance, table = create_table(project_id, instance_id, table_id) + client = bigtable.Client(project=project_id, admin=True) + instance = client.instance(instance_id) + with create_table_cm(project_id, instance_id, table_id, verbose=False) as table: + # [START bigtable_list_tables] + tables = instance.list_tables() + print("Listing tables in current project...") + if tables != []: + for tbl in tables: + print(tbl.table_id) + else: + print("No table exists in current project...") + # [END bigtable_list_tables] + + # [START bigtable_create_family_gc_max_age] + print("Creating column family cf1 with with MaxAge GC Rule...") + # Create a column family with GC policy : maximum age + # where age = current time minus cell timestamp + + # Define the GC rule to retain data with max age of 5 days + max_age_rule = column_family.MaxAgeGCRule(datetime.timedelta(days=5)) + + column_family1 = table.column_family("cf1", max_age_rule) + column_family1.create() + print("Created column family cf1 with MaxAge GC Rule.") + # [END bigtable_create_family_gc_max_age] + + # [START bigtable_create_family_gc_max_versions] + print("Creating column family cf2 with max versions GC rule...") + # Create a column family with GC policy : most recent N versions + # where 1 = most recent version + + # Define the GC policy to retain only the most recent 2 versions + max_versions_rule = column_family.MaxVersionsGCRule(2) + + column_family2 = table.column_family("cf2", max_versions_rule) + column_family2.create() + print("Created column family cf2 with Max Versions GC Rule.") + # [END bigtable_create_family_gc_max_versions] + + # [START bigtable_create_family_gc_union] + print("Creating column family cf3 with union GC rule...") + # Create a column family with GC policy to drop data that matches + # at least one condition. + # Define a GC rule to drop cells older than 5 days or not the + # most recent version + union_rule = column_family.GCRuleUnion( + [ + column_family.MaxAgeGCRule(datetime.timedelta(days=5)), + column_family.MaxVersionsGCRule(2), + ] + ) - # [START bigtable_list_tables] - tables = instance.list_tables() - print("Listing tables in current project...") - if tables != []: - for tbl in tables: - print(tbl.table_id) - else: - print("No table exists in current project...") - # [END bigtable_list_tables] - - # [START bigtable_create_family_gc_max_age] - print("Creating column family cf1 with with MaxAge GC Rule...") - # Create a column family with GC policy : maximum age - # where age = current time minus cell timestamp - - # Define the GC rule to retain data with max age of 5 days - max_age_rule = column_family.MaxAgeGCRule(datetime.timedelta(days=5)) - - column_family1 = table.column_family("cf1", max_age_rule) - column_family1.create() - print("Created column family cf1 with MaxAge GC Rule.") - # [END bigtable_create_family_gc_max_age] - - # [START bigtable_create_family_gc_max_versions] - print("Creating column family cf2 with max versions GC rule...") - # Create a column family with GC policy : most recent N versions - # where 1 = most recent version - - # Define the GC policy to retain only the most recent 2 versions - max_versions_rule = column_family.MaxVersionsGCRule(2) - - column_family2 = table.column_family("cf2", max_versions_rule) - column_family2.create() - print("Created column family cf2 with Max Versions GC Rule.") - # [END bigtable_create_family_gc_max_versions] - - # [START bigtable_create_family_gc_union] - print("Creating column family cf3 with union GC rule...") - # Create a column family with GC policy to drop data that matches - # at least one condition. - # Define a GC rule to drop cells older than 5 days or not the - # most recent version - union_rule = column_family.GCRuleUnion( - [ - column_family.MaxAgeGCRule(datetime.timedelta(days=5)), - column_family.MaxVersionsGCRule(2), - ] - ) + column_family3 = table.column_family("cf3", union_rule) + column_family3.create() + print("Created column family cf3 with Union GC rule") + # [END bigtable_create_family_gc_union] + + # [START bigtable_create_family_gc_intersection] + print("Creating column family cf4 with Intersection GC rule...") + # Create a column family with GC policy to drop data that matches + # all conditions + # GC rule: Drop cells older than 5 days AND older than the most + # recent 2 versions + intersection_rule = column_family.GCRuleIntersection( + [ + column_family.MaxAgeGCRule(datetime.timedelta(days=5)), + column_family.MaxVersionsGCRule(2), + ] + ) - column_family3 = table.column_family("cf3", union_rule) - column_family3.create() - print("Created column family cf3 with Union GC rule") - # [END bigtable_create_family_gc_union] - - # [START bigtable_create_family_gc_intersection] - print("Creating column family cf4 with Intersection GC rule...") - # Create a column family with GC policy to drop data that matches - # all conditions - # GC rule: Drop cells older than 5 days AND older than the most - # recent 2 versions - intersection_rule = column_family.GCRuleIntersection( - [ - column_family.MaxAgeGCRule(datetime.timedelta(days=5)), - column_family.MaxVersionsGCRule(2), - ] - ) + column_family4 = table.column_family("cf4", intersection_rule) + column_family4.create() + print("Created column family cf4 with Intersection GC rule.") + # [END bigtable_create_family_gc_intersection] + + # [START bigtable_create_family_gc_nested] + print("Creating column family cf5 with a Nested GC rule...") + # Create a column family with nested GC policies. + # Create a nested GC rule: + # Drop cells that are either older than the 10 recent versions + # OR + # Drop cells that are older than a month AND older than the + # 2 recent versions + rule1 = column_family.MaxVersionsGCRule(10) + rule2 = column_family.GCRuleIntersection( + [ + column_family.MaxAgeGCRule(datetime.timedelta(days=30)), + column_family.MaxVersionsGCRule(2), + ] + ) - column_family4 = table.column_family("cf4", intersection_rule) - column_family4.create() - print("Created column family cf4 with Intersection GC rule.") - # [END bigtable_create_family_gc_intersection] - - # [START bigtable_create_family_gc_nested] - print("Creating column family cf5 with a Nested GC rule...") - # Create a column family with nested GC policies. - # Create a nested GC rule: - # Drop cells that are either older than the 10 recent versions - # OR - # Drop cells that are older than a month AND older than the - # 2 recent versions - rule1 = column_family.MaxVersionsGCRule(10) - rule2 = column_family.GCRuleIntersection( - [ - column_family.MaxAgeGCRule(datetime.timedelta(days=30)), - column_family.MaxVersionsGCRule(2), - ] - ) + nested_rule = column_family.GCRuleUnion([rule1, rule2]) + + column_family5 = table.column_family("cf5", nested_rule) + column_family5.create() + print("Created column family cf5 with a Nested GC rule.") + # [END bigtable_create_family_gc_nested] + + # [START bigtable_list_column_families] + print("Printing Column Family and GC Rule for all column families...") + column_families = table.list_column_families() + for column_family_name, gc_rule in sorted(column_families.items()): + print("Column Family:", column_family_name) + print("GC Rule:") + print(gc_rule.to_pb()) + # Sample output: + # Column Family: cf4 + # GC Rule: + # gc_rule { + # intersection { + # rules { + # max_age { + # seconds: 432000 + # } + # } + # rules { + # max_num_versions: 2 + # } + # } + # } + # [END bigtable_list_column_families] + + print("Print column family cf1 GC rule before update...") + print("Column Family: cf1") + print(column_family1.to_pb()) + + # [START bigtable_update_gc_rule] + print("Updating column family cf1 GC rule...") + # Update the column family cf1 to update the GC rule + column_family1 = table.column_family("cf1", column_family.MaxVersionsGCRule(1)) + column_family1.update() + print("Updated column family cf1 GC rule\n") + # [END bigtable_update_gc_rule] + + print("Print column family cf1 GC rule after update...") + print("Column Family: cf1") + print(column_family1.to_pb()) + + # [START bigtable_delete_family] + print("Delete a column family cf2...") + # Delete a column family + column_family2.delete() + print("Column family cf2 deleted successfully.") + # [END bigtable_delete_family] - nested_rule = column_family.GCRuleUnion([rule1, rule2]) - - column_family5 = table.column_family("cf5", nested_rule) - column_family5.create() - print("Created column family cf5 with a Nested GC rule.") - # [END bigtable_create_family_gc_nested] - - # [START bigtable_list_column_families] - print("Printing Column Family and GC Rule for all column families...") - column_families = table.list_column_families() - for column_family_name, gc_rule in sorted(column_families.items()): - print("Column Family:", column_family_name) - print("GC Rule:") - print(gc_rule.to_pb()) - # Sample output: - # Column Family: cf4 - # GC Rule: - # gc_rule { - # intersection { - # rules { - # max_age { - # seconds: 432000 - # } - # } - # rules { - # max_num_versions: 2 - # } - # } - # } - # [END bigtable_list_column_families] - - print("Print column family cf1 GC rule before update...") - print("Column Family: cf1") - print(column_family1.to_pb()) - - # [START bigtable_update_gc_rule] - print("Updating column family cf1 GC rule...") - # Update the column family cf1 to update the GC rule - column_family1 = table.column_family("cf1", column_family.MaxVersionsGCRule(1)) - column_family1.update() - print("Updated column family cf1 GC rule\n") - # [END bigtable_update_gc_rule] - - print("Print column family cf1 GC rule after update...") - print("Column Family: cf1") - print(column_family1.to_pb()) - - # [START bigtable_delete_family] - print("Delete a column family cf2...") - # Delete a column family - column_family2.delete() - print("Column family cf2 deleted successfully.") - # [END bigtable_delete_family] - - print( - 'execute command "python tableadmin.py delete [project_id] \ - [instance_id] --table [tableName]" to delete the table.' - ) + print( + 'execute command "python tableadmin.py delete [project_id] \ + [instance_id] --table [tableName]" to delete the table.' + ) def delete_table(project_id, instance_id, table_id): diff --git a/samples/tableadmin/tableadmin_test.py b/samples/tableadmin/tableadmin_test.py index 3063eee9f..0ffdc75c9 100755 --- a/samples/tableadmin/tableadmin_test.py +++ b/samples/tableadmin/tableadmin_test.py @@ -14,29 +14,25 @@ # limitations under the License. import os -import uuid - -from google.api_core import exceptions from test_utils.retry import RetryErrors +from google.api_core import exceptions +import uuid -from tableadmin import create_table -from tableadmin import delete_table -from tableadmin import run_table_operations +from .tableadmin import delete_table +from .tableadmin import run_table_operations +from ..utils import create_table_cm PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] BIGTABLE_INSTANCE = os.environ["BIGTABLE_INSTANCE"] -TABLE_ID_FORMAT = "tableadmin-test-{}" +TABLE_ID = f"tableadmin-test-{str(uuid.uuid4())[:16]}" retry_429_503 = RetryErrors(exceptions.TooManyRequests, exceptions.ServiceUnavailable) def test_run_table_operations(capsys): - table_id = TABLE_ID_FORMAT.format(uuid.uuid4().hex[:8]) - - retry_429_503(run_table_operations)(PROJECT, BIGTABLE_INSTANCE, table_id) + retry_429_503(run_table_operations)(PROJECT, BIGTABLE_INSTANCE, TABLE_ID) out, _ = capsys.readouterr() - assert "Creating the " + table_id + " table." in out assert "Listing tables in current project." in out assert "Creating column family cf1 with with MaxAge GC Rule" in out assert "Created column family cf1 with MaxAge GC Rule." in out @@ -53,14 +49,11 @@ def test_run_table_operations(capsys): assert "Delete a column family cf2..." in out assert "Column family cf2 deleted successfully." in out - retry_429_503(delete_table)(PROJECT, BIGTABLE_INSTANCE, table_id) - def test_delete_table(capsys): - table_id = TABLE_ID_FORMAT.format(uuid.uuid4().hex[:8]) - retry_429_503(create_table)(PROJECT, BIGTABLE_INSTANCE, table_id) - - retry_429_503(delete_table)(PROJECT, BIGTABLE_INSTANCE, table_id) + table_id = f"table-admin-to-delete-{str(uuid.uuid4())[:16]}" + with create_table_cm(PROJECT, BIGTABLE_INSTANCE, table_id, verbose=False): + delete_table(PROJECT, BIGTABLE_INSTANCE, table_id) out, _ = capsys.readouterr() assert "Table " + table_id + " exists." in out diff --git a/samples/utils.py b/samples/utils.py new file mode 100644 index 000000000..eb0ca68f9 --- /dev/null +++ b/samples/utils.py @@ -0,0 +1,87 @@ +# Copyright 2024, Google LLC +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Provides helper logic used across samples +""" + + +from google.cloud import bigtable +from google.api_core import exceptions +from google.api_core.retry import Retry +from google.api_core.retry import if_exception_type + +delete_retry = Retry(if_exception_type(exceptions.TooManyRequests, exceptions.ServiceUnavailable)) + +class create_table_cm: + """ + Create a new table using a context manager, to ensure that table.delete() is called to clean up + the table, even if an exception is thrown + """ + def __init__(self, *args, verbose=True, **kwargs): + self._args = args + self._kwargs = kwargs + self._verbose = verbose + + def __enter__(self): + self._table = create_table(*self._args, **self._kwargs) + if self._verbose: + print(f"created table: {self._table.table_id}") + return self._table + + def __exit__(self, *args): + if self._table.exists(): + if self._verbose: + print(f"deleting table: {self._table.table_id}") + delete_retry(self._table.delete()) + else: + if self._verbose: + print(f"table {self._table.table_id} not found") + + +def create_table(project, instance_id, table_id, column_families={}): + """ + Creates a new table, and blocks until it reaches a ready state + """ + client = bigtable.Client(project=project, admin=True) + instance = client.instance(instance_id) + + table = instance.table(table_id) + if table.exists(): + table.delete() + + kwargs = {} + if column_families: + kwargs["column_families"] = column_families + table.create(**kwargs) + + wait_for_table(table) + + return table + +@Retry( + on_error=if_exception_type( + exceptions.PreconditionFailed, + exceptions.FailedPrecondition, + exceptions.NotFound, + ), + timeout=120, +) +def wait_for_table(table): + """ + raises an exception if the table does not exist or is not ready to use + + Because this method is wrapped with an api_core.Retry decorator, it will + retry with backoff if the table is not ready + """ + if not table.exists(): + raise exceptions.NotFound \ No newline at end of file