From e73d7d46ee58a564b533c113b7ae0e9db90ca51b Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Thu, 7 Feb 2019 10:53:05 -0800 Subject: [PATCH 01/42] BigQuery Storage API sample for reading pandas dataframe [(#1994)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/1994) * BigQuery Storage API sample for reading pandas dataframe How to get a pandas DataFrame, fast! The first two examples use the existing BigQuery client. These examples create a thread pool and read in parallel. The final example shows using just the new BigQuery Storage client, but only shows how to read with a single thread. --- samples/to_dataframe/__init__.py | 0 samples/to_dataframe/main_test.py | 190 ++++++++++++++++++++++++++ samples/to_dataframe/requirements.txt | 5 + 3 files changed, 195 insertions(+) create mode 100644 samples/to_dataframe/__init__.py create mode 100644 samples/to_dataframe/main_test.py create mode 100644 samples/to_dataframe/requirements.txt diff --git a/samples/to_dataframe/__init__.py b/samples/to_dataframe/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/samples/to_dataframe/main_test.py b/samples/to_dataframe/main_test.py new file mode 100644 index 00000000..053bd778 --- /dev/null +++ b/samples/to_dataframe/main_test.py @@ -0,0 +1,190 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import uuid + +import pytest + + +@pytest.fixture +def clients(): + # [START bigquerystorage_pandas_tutorial_all] + # [START bigquerystorage_pandas_tutorial_create_client] + import google.auth + from google.cloud import bigquery + from google.cloud import bigquery_storage_v1beta1 + + # Explicitly create a credentials object. This allows you to use the same + # credentials for both the BigQuery and BigQuery Storage clients, avoiding + # unnecessary API calls to fetch duplicate authentication tokens. + credentials, your_project_id = google.auth.default( + scopes=["https://www.googleapis.com/auth/cloud-platform"] + ) + + # Make clients. + bqclient = bigquery.Client( + credentials=credentials, + project=your_project_id + ) + bqstorageclient = bigquery_storage_v1beta1.BigQueryStorageClient( + credentials=credentials + ) + # [END bigquerystorage_pandas_tutorial_create_client] + # [END bigquerystorage_pandas_tutorial_all] + return bqclient, bqstorageclient + + +def test_table_to_dataframe(capsys, clients): + from google.cloud import bigquery + + bqclient, bqstorageclient = clients + + # [START bigquerystorage_pandas_tutorial_all] + # [START bigquerystorage_pandas_tutorial_read_table] + # Download a table. + table = bigquery.TableReference.from_string( + "bigquery-public-data.utility_us.country_code_iso" + ) + rows = bqclient.list_rows( + table, + selected_fields=[ + bigquery.SchemaField("country_name", "STRING"), + bigquery.SchemaField("fips_code", "STRING"), + ], + ) + dataframe = rows.to_dataframe(bqstorage_client=bqstorageclient) + print(dataframe.head()) + # [END bigquerystorage_pandas_tutorial_read_table] + # [END bigquerystorage_pandas_tutorial_all] + + out, _ = capsys.readouterr() + assert "country_name" in out + + +@pytest.fixture +def temporary_dataset(clients): + from google.cloud import bigquery + + bqclient, _ = clients + + # [START bigquerystorage_pandas_tutorial_all] + # [START bigquerystorage_pandas_tutorial_create_dataset] + # Set the dataset_id to the dataset used to store temporary results. + dataset_id = "query_results_dataset" + # [END bigquerystorage_pandas_tutorial_create_dataset] + # [END bigquerystorage_pandas_tutorial_all] + + dataset_id = "bqstorage_to_dataset_{}".format(uuid.uuid4().hex) + + # [START bigquerystorage_pandas_tutorial_all] + # [START bigquerystorage_pandas_tutorial_create_dataset] + dataset_ref = bqclient.dataset(dataset_id) + dataset = bigquery.Dataset(dataset_ref) + + # Remove tables after 24 hours. + dataset.default_table_expiration_ms = 1000 * 60 * 60 * 24 + + bqclient.create_dataset(dataset) # API request. + # [END bigquerystorage_pandas_tutorial_create_dataset] + # [END bigquerystorage_pandas_tutorial_all] + yield dataset_ref + # [START bigquerystorage_pandas_tutorial_cleanup] + bqclient.delete_dataset(dataset_ref, delete_contents=True) + # [END bigquerystorage_pandas_tutorial_cleanup] + + +def test_query_to_dataframe(capsys, clients, temporary_dataset): + from google.cloud import bigquery + + bqclient, bqstorageclient = clients + dataset_ref = temporary_dataset + + # [START bigquerystorage_pandas_tutorial_all] + # [START bigquerystorage_pandas_tutorial_read_query_results] + import uuid + + # Download query results. + query_string = """ + SELECT + CONCAT( + 'https://stackoverflow.com/questions/', + CAST(id as STRING)) as url, + view_count + FROM `bigquery-public-data.stackoverflow.posts_questions` + WHERE tags like '%google-bigquery%' + ORDER BY view_count DESC + """ + # Use a random table name to avoid overwriting existing tables. + table_id = "queryresults_" + uuid.uuid4().hex + table = dataset_ref.table(table_id) + query_config = bigquery.QueryJobConfig( + # Due to a known issue in the BigQuery Storage API, small query result + # sets cannot be downloaded. To workaround this issue, write results to + # a destination table. + destination=table + ) + + dataframe = ( + bqclient.query(query_string, job_config=query_config) + .result() + .to_dataframe(bqstorage_client=bqstorageclient) + ) + print(dataframe.head()) + # [END bigquerystorage_pandas_tutorial_read_query_results] + # [END bigquerystorage_pandas_tutorial_all] + + out, _ = capsys.readouterr() + assert "stackoverflow" in out + + +def test_session_to_dataframe(capsys, clients): + from google.cloud import bigquery_storage_v1beta1 + + bqclient, bqstorageclient = clients + your_project_id = bqclient.project + + # [START bigquerystorage_pandas_tutorial_all] + # [START bigquerystorage_pandas_tutorial_read_session] + table = bigquery_storage_v1beta1.types.TableReference() + table.project_id = "bigquery-public-data" + table.dataset_id = "new_york_trees" + table.table_id = "tree_species" + + # Select columns to read with read options. If no read options are + # specified, the whole table is read. + read_options = bigquery_storage_v1beta1.types.TableReadOptions() + read_options.selected_fields.append("species_common_name") + read_options.selected_fields.append("fall_color") + + parent = "projects/{}".format(your_project_id) + session = bqstorageclient.create_read_session( + table, parent, read_options=read_options + ) + + # This example reads from only a single stream. Read from multiple streams + # to fetch data faster. Note that the session may not contain any streams + # if there are no rows to read. + stream = session.streams[0] + position = bigquery_storage_v1beta1.types.StreamPosition(stream=stream) + reader = bqstorageclient.read_rows(position) + + # Parse all Avro blocks and create a dataframe. This call requires a + # session, because the session contains the schema for the row blocks. + dataframe = reader.to_dataframe(session) + print(dataframe.head()) + # [END bigquerystorage_pandas_tutorial_read_session] + # [END bigquerystorage_pandas_tutorial_all] + + out, _ = capsys.readouterr() + assert "species_common_name" in out diff --git a/samples/to_dataframe/requirements.txt b/samples/to_dataframe/requirements.txt new file mode 100644 index 00000000..29d1de55 --- /dev/null +++ b/samples/to_dataframe/requirements.txt @@ -0,0 +1,5 @@ +google-auth==1.6.2 +google-cloud-bigquery-storage==0.2.0 +google-cloud-bigquery==1.8.1 +fastavro==0.21.17 +pandas==0.24.0 \ No newline at end of file From 9d1a280e455b18f0956745fb4cb0e35c2fbf85fc Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Thu, 4 Apr 2019 12:29:03 -0700 Subject: [PATCH 02/42] Remove temporary dataset from bqstorage pandas tutorial [(#2088)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/2088) * Remove temporary dataset from bqstorage pandas tutorial As of google-cloud-bigquery version 1.11.1, the `to_dataframe` method will fallback to the tabledata.list API when the BigQuery Storage API fails to read the query results. * Remove unused imports --- samples/to_dataframe/main_test.py | 57 ++++----------------------- samples/to_dataframe/requirements.txt | 4 +- 2 files changed, 9 insertions(+), 52 deletions(-) diff --git a/samples/to_dataframe/main_test.py b/samples/to_dataframe/main_test.py index 053bd778..586ab3f9 100644 --- a/samples/to_dataframe/main_test.py +++ b/samples/to_dataframe/main_test.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import uuid - import pytest @@ -72,48 +70,11 @@ def test_table_to_dataframe(capsys, clients): assert "country_name" in out -@pytest.fixture -def temporary_dataset(clients): - from google.cloud import bigquery - - bqclient, _ = clients - - # [START bigquerystorage_pandas_tutorial_all] - # [START bigquerystorage_pandas_tutorial_create_dataset] - # Set the dataset_id to the dataset used to store temporary results. - dataset_id = "query_results_dataset" - # [END bigquerystorage_pandas_tutorial_create_dataset] - # [END bigquerystorage_pandas_tutorial_all] - - dataset_id = "bqstorage_to_dataset_{}".format(uuid.uuid4().hex) - - # [START bigquerystorage_pandas_tutorial_all] - # [START bigquerystorage_pandas_tutorial_create_dataset] - dataset_ref = bqclient.dataset(dataset_id) - dataset = bigquery.Dataset(dataset_ref) - - # Remove tables after 24 hours. - dataset.default_table_expiration_ms = 1000 * 60 * 60 * 24 - - bqclient.create_dataset(dataset) # API request. - # [END bigquerystorage_pandas_tutorial_create_dataset] - # [END bigquerystorage_pandas_tutorial_all] - yield dataset_ref - # [START bigquerystorage_pandas_tutorial_cleanup] - bqclient.delete_dataset(dataset_ref, delete_contents=True) - # [END bigquerystorage_pandas_tutorial_cleanup] - - -def test_query_to_dataframe(capsys, clients, temporary_dataset): - from google.cloud import bigquery - +def test_query_to_dataframe(capsys, clients): bqclient, bqstorageclient = clients - dataset_ref = temporary_dataset # [START bigquerystorage_pandas_tutorial_all] # [START bigquerystorage_pandas_tutorial_read_query_results] - import uuid - # Download query results. query_string = """ SELECT @@ -125,19 +86,15 @@ def test_query_to_dataframe(capsys, clients, temporary_dataset): WHERE tags like '%google-bigquery%' ORDER BY view_count DESC """ - # Use a random table name to avoid overwriting existing tables. - table_id = "queryresults_" + uuid.uuid4().hex - table = dataset_ref.table(table_id) - query_config = bigquery.QueryJobConfig( - # Due to a known issue in the BigQuery Storage API, small query result - # sets cannot be downloaded. To workaround this issue, write results to - # a destination table. - destination=table - ) dataframe = ( - bqclient.query(query_string, job_config=query_config) + bqclient.query(query_string) .result() + + # Note: The BigQuery Storage API cannot be used to download small query + # results, but as of google-cloud-bigquery version 1.11.1, the + # to_dataframe method will fallback to the tabledata.list API when the + # BigQuery Storage API fails to read the query results. .to_dataframe(bqstorage_client=bqstorageclient) ) print(dataframe.head()) diff --git a/samples/to_dataframe/requirements.txt b/samples/to_dataframe/requirements.txt index 29d1de55..24b2a546 100644 --- a/samples/to_dataframe/requirements.txt +++ b/samples/to_dataframe/requirements.txt @@ -1,5 +1,5 @@ google-auth==1.6.2 -google-cloud-bigquery-storage==0.2.0 -google-cloud-bigquery==1.8.1 +google-cloud-bigquery-storage==0.3.0 +google-cloud-bigquery==1.11.1 fastavro==0.21.17 pandas==0.24.0 \ No newline at end of file From da2839762e041afba1216b7105c63efbcb6d4476 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Tue, 9 Apr 2019 14:13:10 -0700 Subject: [PATCH 03/42] Add magics tutorial with BigQuery Storage API integration. [(#2087)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/2087) * Add magics tutorial with BigQuery Storage API integration. This is a notebooks tutorial, modeled after the Jupyter notebook example code for BigQuery. Use some caution when running these tests, as they run some large-ish (5 GB processed) queries and download about 500 MB worth of data. This is intentional, as the BigQuery Storage API is most useful for downloading large results. * Update deps. * Don't run big queries on Travis. --- samples/to_dataframe/jupyter_test.py | 148 ++++++++++++++++++++++++++ samples/to_dataframe/requirements.txt | 1 + 2 files changed, 149 insertions(+) create mode 100644 samples/to_dataframe/jupyter_test.py diff --git a/samples/to_dataframe/jupyter_test.py b/samples/to_dataframe/jupyter_test.py new file mode 100644 index 00000000..027d0c7d --- /dev/null +++ b/samples/to_dataframe/jupyter_test.py @@ -0,0 +1,148 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import IPython +from IPython.terminal import interactiveshell +from IPython.testing import tools +import pytest + +# Ignore semicolon lint warning because semicolons are used in notebooks +# flake8: noqa E703 + + +@pytest.fixture(scope="session") +def ipython(): + config = tools.default_config() + config.TerminalInteractiveShell.simple_prompt = True + shell = interactiveshell.TerminalInteractiveShell.instance(config=config) + return shell + + +@pytest.fixture() +def ipython_interactive(request, ipython): + """Activate IPython's builtin hooks + + for the duration of the test scope. + """ + with ipython.builtin_trap: + yield ipython + + +def _strip_region_tags(sample_text): + """Remove blank lines and region tags from sample text""" + magic_lines = [ + line for line in sample_text.split("\n") if len(line) > 0 and "# [" not in line + ] + return "\n".join(magic_lines) + + +def test_jupyter_small_query(ipython): + ip = IPython.get_ipython() + ip.extension_manager.load_extension("google.cloud.bigquery") + + # Include a small query to demonstrate that it falls back to the + # tabledata.list API when the BQ Storage API cannot be used. + sample = """ + # [START bigquerystorage_jupyter_tutorial_fallback] + %%bigquery stackoverflow --use_bqstorage_api + SELECT + CONCAT( + 'https://stackoverflow.com/questions/', + CAST(id as STRING)) as url, + view_count + FROM `bigquery-public-data.stackoverflow.posts_questions` + WHERE tags like '%google-bigquery%' + ORDER BY view_count DESC + LIMIT 10 + # [END bigquerystorage_jupyter_tutorial_fallback] + """ + + result = ip.run_cell(_strip_region_tags(sample)) + result.raise_error() # Throws an exception if the cell failed. + assert "stackoverflow" in ip.user_ns # verify that variable exists + + +@pytest.mark.skipif( + "TRAVIS" in os.environ, reason="Not running long-running queries on Travis" +) +def test_jupyter_tutorial(ipython): + ip = IPython.get_ipython() + ip.extension_manager.load_extension("google.cloud.bigquery") + + # This code sample intentionally queries a lot of data to demonstrate the + # speed-up of using the BigQuery Storage API to download the results. + sample = """ + # [START bigquerystorage_jupyter_tutorial_query] + %%bigquery nodejs_deps --use_bqstorage_api + SELECT + dependency_name, + dependency_platform, + project_name, + project_id, + version_number, + version_id, + dependency_kind, + optional_dependency, + dependency_requirements, + dependency_project_id + FROM + `bigquery-public-data.libraries_io.dependencies` + WHERE + LOWER(dependency_platform) = 'npm' + LIMIT 2500000 + # [END bigquerystorage_jupyter_tutorial_query] + """ + result = ip.run_cell(_strip_region_tags(sample)) + result.raise_error() # Throws an exception if the cell failed. + + assert "nodejs_deps" in ip.user_ns # verify that variable exists + nodejs_deps = ip.user_ns["nodejs_deps"] + + # [START bigquerystorage_jupyter_tutorial_results] + nodejs_deps.head() + # [END bigquerystorage_jupyter_tutorial_results] + + # [START bigquerystorage_jupyter_tutorial_context] + import google.cloud.bigquery.magics + + google.cloud.bigquery.magics.context.use_bqstorage_api = True + # [END bigquerystorage_jupyter_tutorial_context] + + sample = """ + # [START bigquerystorage_jupyter_tutorial_query] + %%bigquery java_deps + SELECT + dependency_name, + dependency_platform, + project_name, + project_id, + version_number, + version_id, + dependency_kind, + optional_dependency, + dependency_requirements, + dependency_project_id + FROM + `bigquery-public-data.libraries_io.dependencies` + WHERE + LOWER(dependency_platform) = 'maven' + LIMIT 2500000 + # [END bigquerystorage_jupyter_tutorial_query] + """ + result = ip.run_cell(_strip_region_tags(sample)) + result.raise_error() # Throws an exception if the cell failed. + + assert "java_deps" in ip.user_ns # verify that variable exists diff --git a/samples/to_dataframe/requirements.txt b/samples/to_dataframe/requirements.txt index 24b2a546..2fab8850 100644 --- a/samples/to_dataframe/requirements.txt +++ b/samples/to_dataframe/requirements.txt @@ -2,4 +2,5 @@ google-auth==1.6.2 google-cloud-bigquery-storage==0.3.0 google-cloud-bigquery==1.11.1 fastavro==0.21.17 +ipython==7.2.0 pandas==0.24.0 \ No newline at end of file From ce20d43976c2abd4dc9ac7278f79ad5e0fc1658b Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Tue, 9 Apr 2019 14:59:16 -0700 Subject: [PATCH 04/42] Add bigquerystorage_jupyter_tutorial_query_default region tag. --- samples/to_dataframe/jupyter_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/samples/to_dataframe/jupyter_test.py b/samples/to_dataframe/jupyter_test.py index 027d0c7d..ef1b0ddb 100644 --- a/samples/to_dataframe/jupyter_test.py +++ b/samples/to_dataframe/jupyter_test.py @@ -122,7 +122,7 @@ def test_jupyter_tutorial(ipython): # [END bigquerystorage_jupyter_tutorial_context] sample = """ - # [START bigquerystorage_jupyter_tutorial_query] + # [START bigquerystorage_jupyter_tutorial_query_default] %%bigquery java_deps SELECT dependency_name, @@ -140,7 +140,7 @@ def test_jupyter_tutorial(ipython): WHERE LOWER(dependency_platform) = 'maven' LIMIT 2500000 - # [END bigquerystorage_jupyter_tutorial_query] + # [END bigquerystorage_jupyter_tutorial_query_default] """ result = ip.run_cell(_strip_region_tags(sample)) result.raise_error() # Throws an exception if the cell failed. From db2b426f5f9c1c3ab725b51760ca36ba7f560173 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Fri, 12 Jul 2019 12:37:21 -0500 Subject: [PATCH 05/42] BigQuery Storage: Update to use faster Arrow data format. [(#2269)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/2269) --- samples/to_dataframe/main_test.py | 20 +++++++++++++------- samples/to_dataframe/requirements.txt | 8 ++++---- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/samples/to_dataframe/main_test.py b/samples/to_dataframe/main_test.py index 586ab3f9..8335b437 100644 --- a/samples/to_dataframe/main_test.py +++ b/samples/to_dataframe/main_test.py @@ -33,7 +33,7 @@ def clients(): # Make clients. bqclient = bigquery.Client( credentials=credentials, - project=your_project_id + project=your_project_id, ) bqstorageclient = bigquery_storage_v1beta1.BigQueryStorageClient( credentials=credentials @@ -90,11 +90,6 @@ def test_query_to_dataframe(capsys, clients): dataframe = ( bqclient.query(query_string) .result() - - # Note: The BigQuery Storage API cannot be used to download small query - # results, but as of google-cloud-bigquery version 1.11.1, the - # to_dataframe method will fallback to the tabledata.list API when the - # BigQuery Storage API fails to read the query results. .to_dataframe(bqstorage_client=bqstorageclient) ) print(dataframe.head()) @@ -126,7 +121,18 @@ def test_session_to_dataframe(capsys, clients): parent = "projects/{}".format(your_project_id) session = bqstorageclient.create_read_session( - table, parent, read_options=read_options + table, + parent, + read_options=read_options, + # This API can also deliver data serialized in Apache Avro format. + # This example leverages Apache Arrow. + format_=bigquery_storage_v1beta1.enums.DataFormat.ARROW, + # We use a LIQUID strategy in this example because we only read from a + # single stream. Consider BALANCED if you're consuming multiple streams + # concurrently and want more consistent stream sizes. + sharding_strategy=( + bigquery_storage_v1beta1.enums.ShardingStrategy.LIQUID + ), ) # This example reads from only a single stream. Read from multiple streams diff --git a/samples/to_dataframe/requirements.txt b/samples/to_dataframe/requirements.txt index 2fab8850..5dad9dad 100644 --- a/samples/to_dataframe/requirements.txt +++ b/samples/to_dataframe/requirements.txt @@ -1,6 +1,6 @@ google-auth==1.6.2 -google-cloud-bigquery-storage==0.3.0 -google-cloud-bigquery==1.11.1 -fastavro==0.21.17 +google-cloud-bigquery-storage==0.6.0 +google-cloud-bigquery==1.17.0 +pyarrow==0.13.0 ipython==7.2.0 -pandas==0.24.0 \ No newline at end of file +pandas==0.24.2 \ No newline at end of file From 0b942504c847c30b0b3a328b4a4fc0b538bbec83 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Tue, 24 Sep 2019 15:55:51 -0700 Subject: [PATCH 06/42] Update pandas/bqstorage samples to latest library changes. [(#2413)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/2413) --- samples/to_dataframe/jupyter_test.py | 49 +++++---------------------- samples/to_dataframe/requirements.txt | 8 ++--- 2 files changed, 12 insertions(+), 45 deletions(-) diff --git a/samples/to_dataframe/jupyter_test.py b/samples/to_dataframe/jupyter_test.py index ef1b0ddb..7997ee2e 100644 --- a/samples/to_dataframe/jupyter_test.py +++ b/samples/to_dataframe/jupyter_test.py @@ -75,9 +75,6 @@ def test_jupyter_small_query(ipython): assert "stackoverflow" in ip.user_ns # verify that variable exists -@pytest.mark.skipif( - "TRAVIS" in os.environ, reason="Not running long-running queries on Travis" -) def test_jupyter_tutorial(ipython): ip = IPython.get_ipython() ip.extension_manager.load_extension("google.cloud.bigquery") @@ -86,33 +83,18 @@ def test_jupyter_tutorial(ipython): # speed-up of using the BigQuery Storage API to download the results. sample = """ # [START bigquerystorage_jupyter_tutorial_query] - %%bigquery nodejs_deps --use_bqstorage_api - SELECT - dependency_name, - dependency_platform, - project_name, - project_id, - version_number, - version_id, - dependency_kind, - optional_dependency, - dependency_requirements, - dependency_project_id - FROM - `bigquery-public-data.libraries_io.dependencies` - WHERE - LOWER(dependency_platform) = 'npm' - LIMIT 2500000 + %%bigquery tax_forms --use_bqstorage_api + SELECT * FROM `bigquery-public-data.irs_990.irs_990_2012` # [END bigquerystorage_jupyter_tutorial_query] """ result = ip.run_cell(_strip_region_tags(sample)) result.raise_error() # Throws an exception if the cell failed. - assert "nodejs_deps" in ip.user_ns # verify that variable exists - nodejs_deps = ip.user_ns["nodejs_deps"] + assert "tax_forms" in ip.user_ns # verify that variable exists + tax_forms = ip.user_ns["tax_forms"] # [START bigquerystorage_jupyter_tutorial_results] - nodejs_deps.head() + tax_forms.head() # [END bigquerystorage_jupyter_tutorial_results] # [START bigquerystorage_jupyter_tutorial_context] @@ -123,26 +105,11 @@ def test_jupyter_tutorial(ipython): sample = """ # [START bigquerystorage_jupyter_tutorial_query_default] - %%bigquery java_deps - SELECT - dependency_name, - dependency_platform, - project_name, - project_id, - version_number, - version_id, - dependency_kind, - optional_dependency, - dependency_requirements, - dependency_project_id - FROM - `bigquery-public-data.libraries_io.dependencies` - WHERE - LOWER(dependency_platform) = 'maven' - LIMIT 2500000 + %%bigquery tax_forms + SELECT * FROM `bigquery-public-data.irs_990.irs_990_2012` # [END bigquerystorage_jupyter_tutorial_query_default] """ result = ip.run_cell(_strip_region_tags(sample)) result.raise_error() # Throws an exception if the cell failed. - assert "java_deps" in ip.user_ns # verify that variable exists + assert "tax_forms" in ip.user_ns # verify that variable exists diff --git a/samples/to_dataframe/requirements.txt b/samples/to_dataframe/requirements.txt index 5dad9dad..d5a1d3b5 100644 --- a/samples/to_dataframe/requirements.txt +++ b/samples/to_dataframe/requirements.txt @@ -1,6 +1,6 @@ google-auth==1.6.2 -google-cloud-bigquery-storage==0.6.0 -google-cloud-bigquery==1.17.0 -pyarrow==0.13.0 +google-cloud-bigquery-storage==0.7.0 +google-cloud-bigquery==1.20.0 +pyarrow==0.14.1 ipython==7.2.0 -pandas==0.24.2 \ No newline at end of file +pandas==0.25.1 \ No newline at end of file From 3ac2b1af87145543fe4067eb55940f96a54e6f21 Mon Sep 17 00:00:00 2001 From: Gus Class Date: Wed, 23 Oct 2019 16:27:00 -0700 Subject: [PATCH 07/42] Adds updates including compute [(#2436)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/2436) * Adds updates including compute * Python 2 compat pytest * Fixing weird \r\n issue from GH merge * Put asset tests back in * Re-add pod operator test * Hack parameter for k8s pod operator --- samples/to_dataframe/requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/samples/to_dataframe/requirements.txt b/samples/to_dataframe/requirements.txt index d5a1d3b5..5b42cef1 100644 --- a/samples/to_dataframe/requirements.txt +++ b/samples/to_dataframe/requirements.txt @@ -1,6 +1,6 @@ -google-auth==1.6.2 +google-auth==1.6.3 google-cloud-bigquery-storage==0.7.0 google-cloud-bigquery==1.20.0 pyarrow==0.14.1 -ipython==7.2.0 -pandas==0.25.1 \ No newline at end of file +ipython==7.8.0 +pandas==0.25.1 From 31118c59257fbb1b7888e8ee297490b9e7c8a485 Mon Sep 17 00:00:00 2001 From: DPEBot Date: Fri, 20 Dec 2019 17:41:38 -0800 Subject: [PATCH 08/42] Auto-update dependencies. [(#2005)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/2005) * Auto-update dependencies. * Revert update of appengine/flexible/datastore. * revert update of appengine/flexible/scipy * revert update of bigquery/bqml * revert update of bigquery/cloud-client * revert update of bigquery/datalab-migration * revert update of bigtable/quickstart * revert update of compute/api * revert update of container_registry/container_analysis * revert update of dataflow/run_template * revert update of datastore/cloud-ndb * revert update of dialogflow/cloud-client * revert update of dlp * revert update of functions/imagemagick * revert update of functions/ocr/app * revert update of healthcare/api-client/fhir * revert update of iam/api-client * revert update of iot/api-client/gcs_file_to_device * revert update of iot/api-client/mqtt_example * revert update of language/automl * revert update of run/image-processing * revert update of vision/automl * revert update testing/requirements.txt * revert update of vision/cloud-client/detect * revert update of vision/cloud-client/product_search * revert update of jobs/v2/api_client * revert update of jobs/v3/api_client * revert update of opencensus * revert update of translate/cloud-client * revert update to speech/cloud-client Co-authored-by: Kurtis Van Gent <31518063+kurtisvg@users.noreply.github.com> Co-authored-by: Doug Mahugh --- samples/to_dataframe/requirements.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/samples/to_dataframe/requirements.txt b/samples/to_dataframe/requirements.txt index 5b42cef1..d513f1b7 100644 --- a/samples/to_dataframe/requirements.txt +++ b/samples/to_dataframe/requirements.txt @@ -1,6 +1,6 @@ -google-auth==1.6.3 +google-auth==1.10.0 google-cloud-bigquery-storage==0.7.0 -google-cloud-bigquery==1.20.0 -pyarrow==0.14.1 -ipython==7.8.0 -pandas==0.25.1 +google-cloud-bigquery==1.23.1 +pyarrow==0.15.1 +ipython==7.10.2 +pandas==0.25.3 From 4bfc8419f7b46553a1db3829def61a9dcb0fd01d Mon Sep 17 00:00:00 2001 From: "Leah E. Cole" <6719667+leahecole@users.noreply.github.com> Date: Thu, 5 Mar 2020 14:22:12 -0800 Subject: [PATCH 09/42] chore(deps): update dependency google-auth to v1.11.2 [(#2724)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/2724) Co-authored-by: Leah E. Cole <6719667+leahecole@users.noreply.github.com> --- samples/to_dataframe/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/to_dataframe/requirements.txt b/samples/to_dataframe/requirements.txt index d513f1b7..16905131 100644 --- a/samples/to_dataframe/requirements.txt +++ b/samples/to_dataframe/requirements.txt @@ -1,4 +1,4 @@ -google-auth==1.10.0 +google-auth==1.11.2 google-cloud-bigquery-storage==0.7.0 google-cloud-bigquery==1.23.1 pyarrow==0.15.1 From 4c1ffdf903ea6da29d86d887c4c7d90cdeab7213 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Fri, 6 Mar 2020 21:48:04 +0100 Subject: [PATCH 10/42] chore(deps): update dependency google-cloud-bigquery-storage to v0.8.0 [(#3050)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/3050) * chore(deps): update dependency google-cloud-bigquery-storage to v0.8.0 * chore(deps): update pandas-gbq * chore(deps): update ipython * chore: update requirements.txt * chore: it is spelled version. * chore(deps): split pandas version * chore(deps): split pandas version Co-authored-by: Christopher Wilcox Co-authored-by: Leah Cole --- samples/to_dataframe/requirements.txt | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/samples/to_dataframe/requirements.txt b/samples/to_dataframe/requirements.txt index 16905131..bb2be214 100644 --- a/samples/to_dataframe/requirements.txt +++ b/samples/to_dataframe/requirements.txt @@ -1,6 +1,8 @@ google-auth==1.11.2 -google-cloud-bigquery-storage==0.7.0 +google-cloud-bigquery-storage==0.8.0 google-cloud-bigquery==1.23.1 pyarrow==0.15.1 -ipython==7.10.2 -pandas==0.25.3 +ipython==7.10.2; python_version > '3.0' +ipython==5.9.0; python_version < '3.0' +pandas==0.25.3; python_version > '3.0' +pandas==0.24.2; python_version < '3.0' From eddb7bf33b224ab2f75d379f977b2566f57d3c79 Mon Sep 17 00:00:00 2001 From: Kurtis Van Gent <31518063+kurtisvg@users.noreply.github.com> Date: Wed, 1 Apr 2020 19:11:50 -0700 Subject: [PATCH 11/42] Simplify noxfile setup. [(#2806)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/2806) * chore(deps): update dependency requests to v2.23.0 * Simplify noxfile and add version control. * Configure appengine/standard to only test Python 2.7. * Update Kokokro configs to match noxfile. * Add requirements-test to each folder. * Remove Py2 versions from everything execept appengine/standard. * Remove conftest.py. * Remove appengine/standard/conftest.py * Remove 'no-sucess-flaky-report' from pytest.ini. * Add GAE SDK back to appengine/standard tests. * Fix typo. * Roll pytest to python 2 version. * Add a bunch of testing requirements. * Remove typo. * Add appengine lib directory back in. * Add some additional requirements. * Fix issue with flake8 args. * Even more requirements. * Readd appengine conftest.py. * Add a few more requirements. * Even more Appengine requirements. * Add webtest for appengine/standard/mailgun. * Add some additional requirements. * Add workaround for issue with mailjet-rest. * Add responses for appengine/standard/mailjet. Co-authored-by: Renovate Bot --- samples/to_dataframe/requirements-test.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 samples/to_dataframe/requirements-test.txt diff --git a/samples/to_dataframe/requirements-test.txt b/samples/to_dataframe/requirements-test.txt new file mode 100644 index 00000000..781d4326 --- /dev/null +++ b/samples/to_dataframe/requirements-test.txt @@ -0,0 +1 @@ +pytest==5.3.2 From f87f1ba4414e16960f4374f707099e84228e7e90 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Fri, 17 Apr 2020 03:09:45 +0200 Subject: [PATCH 12/42] Update dependency google-auth to v1.14.0 [(#3148)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/3148) Co-authored-by: Leah E. Cole <6719667+leahecole@users.noreply.github.com> --- samples/to_dataframe/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/to_dataframe/requirements.txt b/samples/to_dataframe/requirements.txt index bb2be214..070e5d81 100644 --- a/samples/to_dataframe/requirements.txt +++ b/samples/to_dataframe/requirements.txt @@ -1,4 +1,4 @@ -google-auth==1.11.2 +google-auth==1.14.0 google-cloud-bigquery-storage==0.8.0 google-cloud-bigquery==1.23.1 pyarrow==0.15.1 From 4d737475839abdc6f7d0eb1faac3246e6a6df441 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Sat, 18 Apr 2020 07:41:12 +0200 Subject: [PATCH 13/42] chore(deps): update dependency google-cloud-bigquery to v1.24.0 [(#3049)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/3049) * chore(deps): update dependency google-cloud-bigquery to v1.24.0 * chore(deps): update ipython version * fix: fix requirements order * explicitly add grpc to resolve errors * adjust arguments * undo mistake * bump auth version Co-authored-by: Bu Sun Kim <8822365+busunkim96@users.noreply.github.com> Co-authored-by: Leah Cole Co-authored-by: Leah E. Cole <6719667+leahecole@users.noreply.github.com> Co-authored-by: Christopher Wilcox --- samples/to_dataframe/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/to_dataframe/requirements.txt b/samples/to_dataframe/requirements.txt index 070e5d81..37431c62 100644 --- a/samples/to_dataframe/requirements.txt +++ b/samples/to_dataframe/requirements.txt @@ -1,6 +1,6 @@ google-auth==1.14.0 google-cloud-bigquery-storage==0.8.0 -google-cloud-bigquery==1.23.1 +google-cloud-bigquery==1.24.0 pyarrow==0.15.1 ipython==7.10.2; python_version > '3.0' ipython==5.9.0; python_version < '3.0' From 884fead06c08cb9a5469e5061e5be0c0a34309b8 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Sat, 18 Apr 2020 07:47:33 +0200 Subject: [PATCH 14/42] chore(deps): update dependency google-auth-oauthlib to v0.4.1 [(#2797)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/2797) * chore(deps): update dependency google-auth-oauthlib to v0.4.1 * resolve dependency finding errors * fix new matplotlib error Co-authored-by: Leah E. Cole <6719667+leahecole@users.noreply.github.com> Co-authored-by: Leah Cole Co-authored-by: Bu Sun Kim <8822365+busunkim96@users.noreply.github.com> Co-authored-by: Christopher Wilcox From 520a38af7ba4a0f58876db1821a96e5fd8391589 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Wed, 22 Apr 2020 05:40:40 +0200 Subject: [PATCH 15/42] Update dependency pyarrow to v0.17.0 [(#3188)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/3188) Co-authored-by: Leah E. Cole <6719667+leahecole@users.noreply.github.com> --- samples/to_dataframe/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/to_dataframe/requirements.txt b/samples/to_dataframe/requirements.txt index 37431c62..b10d9489 100644 --- a/samples/to_dataframe/requirements.txt +++ b/samples/to_dataframe/requirements.txt @@ -1,7 +1,7 @@ google-auth==1.14.0 google-cloud-bigquery-storage==0.8.0 google-cloud-bigquery==1.24.0 -pyarrow==0.15.1 +pyarrow==0.17.0 ipython==7.10.2; python_version > '3.0' ipython==5.9.0; python_version < '3.0' pandas==0.25.3; python_version > '3.0' From 15e305262088ce0ff3aa1a569f1acec515a4edf8 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Tue, 28 Apr 2020 06:20:12 +0200 Subject: [PATCH 16/42] chore(deps): update dependency google-auth to v1.14.1 [(#3464)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/3464) This PR contains the following updates: | Package | Update | Change | |---|---|---| | [google-auth](https://togithub.com/googleapis/google-auth-library-python) | patch | `==1.14.0` -> `==1.14.1` | | [google-auth](https://togithub.com/googleapis/google-auth-library-python) | minor | `==1.11.2` -> `==1.14.1` | --- ### Release Notes
googleapis/google-auth-library-python ### [`v1.14.1`](https://togithub.com/googleapis/google-auth-library-python/blob/master/CHANGELOG.md#​1141-httpswwwgithubcomgoogleapisgoogle-auth-library-pythoncomparev1140v1141-2020-04-21) [Compare Source](https://togithub.com/googleapis/google-auth-library-python/compare/v1.14.0...v1.14.1)
--- ### Renovate configuration :date: **Schedule**: At any time (no schedule defined). :vertical_traffic_light: **Automerge**: Disabled by config. Please merge this manually once you are satisfied. :recycle: **Rebasing**: Never, or you tick the rebase/retry checkbox. :no_bell: **Ignore**: Close this PR and you won't be reminded about these updates again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR has been generated by [WhiteSource Renovate](https://renovate.whitesourcesoftware.com). View repository job log [here](https://app.renovatebot.com/dashboard#GoogleCloudPlatform/python-docs-samples). --- samples/to_dataframe/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/to_dataframe/requirements.txt b/samples/to_dataframe/requirements.txt index b10d9489..02b6e217 100644 --- a/samples/to_dataframe/requirements.txt +++ b/samples/to_dataframe/requirements.txt @@ -1,4 +1,4 @@ -google-auth==1.14.0 +google-auth==1.14.1 google-cloud-bigquery-storage==0.8.0 google-cloud-bigquery==1.24.0 pyarrow==0.17.0 From 4a23fbcb801b122581255c56a6035d4615751712 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Mon, 11 May 2020 22:24:11 +0200 Subject: [PATCH 17/42] chore(deps): update dependency google-auth to v1.14.2 [(#3724)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/3724) This PR contains the following updates: | Package | Update | Change | |---|---|---| | [google-auth](https://togithub.com/googleapis/google-auth-library-python) | patch | `==1.14.1` -> `==1.14.2` | --- ### Release Notes
googleapis/google-auth-library-python ### [`v1.14.2`](https://togithub.com/googleapis/google-auth-library-python/blob/master/CHANGELOG.md#​1142-httpswwwgithubcomgoogleapisgoogle-auth-library-pythoncomparev1141v1142-2020-05-07) [Compare Source](https://togithub.com/googleapis/google-auth-library-python/compare/v1.14.1...v1.14.2)
--- ### Renovate configuration :date: **Schedule**: At any time (no schedule defined). :vertical_traffic_light: **Automerge**: Disabled by config. Please merge this manually once you are satisfied. :recycle: **Rebasing**: Never, or you tick the rebase/retry checkbox. :no_bell: **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR has been generated by [WhiteSource Renovate](https://renovate.whitesourcesoftware.com). View repository job log [here](https://app.renovatebot.com/dashboard#GoogleCloudPlatform/python-docs-samples). --- samples/to_dataframe/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/to_dataframe/requirements.txt b/samples/to_dataframe/requirements.txt index 02b6e217..4ad6f3ee 100644 --- a/samples/to_dataframe/requirements.txt +++ b/samples/to_dataframe/requirements.txt @@ -1,4 +1,4 @@ -google-auth==1.14.1 +google-auth==1.14.2 google-cloud-bigquery-storage==0.8.0 google-cloud-bigquery==1.24.0 pyarrow==0.17.0 From 3b67924e2966b1d6991102d97947aa0ac3d2c6db Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Wed, 13 May 2020 08:16:04 +0200 Subject: [PATCH 18/42] chore(deps): update dependency google-auth to v1.14.3 [(#3728)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/3728) This PR contains the following updates: | Package | Update | Change | |---|---|---| | [google-auth](https://togithub.com/googleapis/google-auth-library-python) | patch | `==1.14.2` -> `==1.14.3` | --- ### Release Notes
googleapis/google-auth-library-python ### [`v1.14.3`](https://togithub.com/googleapis/google-auth-library-python/blob/master/CHANGELOG.md#​1143-httpswwwgithubcomgoogleapisgoogle-auth-library-pythoncomparev1142v1143-2020-05-11) [Compare Source](https://togithub.com/googleapis/google-auth-library-python/compare/v1.14.2...v1.14.3)
--- ### Renovate configuration :date: **Schedule**: At any time (no schedule defined). :vertical_traffic_light: **Automerge**: Disabled by config. Please merge this manually once you are satisfied. :recycle: **Rebasing**: Never, or you tick the rebase/retry checkbox. :no_bell: **Ignore**: Close this PR and you won't be reminded about this update again. --- - [x] If you want to rebase/retry this PR, check this box --- This PR has been generated by [WhiteSource Renovate](https://renovate.whitesourcesoftware.com). View repository job log [here](https://app.renovatebot.com/dashboard#GoogleCloudPlatform/python-docs-samples). --- samples/to_dataframe/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/to_dataframe/requirements.txt b/samples/to_dataframe/requirements.txt index 4ad6f3ee..5ed5dc2f 100644 --- a/samples/to_dataframe/requirements.txt +++ b/samples/to_dataframe/requirements.txt @@ -1,4 +1,4 @@ -google-auth==1.14.2 +google-auth==1.14.3 google-cloud-bigquery-storage==0.8.0 google-cloud-bigquery==1.24.0 pyarrow==0.17.0 From b4e0be6c292f62cc509c2714a361a122ca620875 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Mon, 18 May 2020 23:47:52 +0200 Subject: [PATCH 19/42] chore(deps): update dependency pyarrow to v0.17.1 [(#3806)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/3806) --- samples/to_dataframe/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/to_dataframe/requirements.txt b/samples/to_dataframe/requirements.txt index 5ed5dc2f..7a90a31a 100644 --- a/samples/to_dataframe/requirements.txt +++ b/samples/to_dataframe/requirements.txt @@ -1,7 +1,7 @@ google-auth==1.14.3 google-cloud-bigquery-storage==0.8.0 google-cloud-bigquery==1.24.0 -pyarrow==0.17.0 +pyarrow==0.17.1 ipython==7.10.2; python_version > '3.0' ipython==5.9.0; python_version < '3.0' pandas==0.25.3; python_version > '3.0' From 8481d861e443b7d8105acbea7cbeca738efa828c Mon Sep 17 00:00:00 2001 From: "Leah E. Cole" <6719667+leahecole@users.noreply.github.com> Date: Mon, 18 May 2020 20:33:45 -0700 Subject: [PATCH 20/42] update google-auth to 1.15.0 part 2 [(#3815)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/3815) --- samples/to_dataframe/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/to_dataframe/requirements.txt b/samples/to_dataframe/requirements.txt index 7a90a31a..151995f8 100644 --- a/samples/to_dataframe/requirements.txt +++ b/samples/to_dataframe/requirements.txt @@ -1,4 +1,4 @@ -google-auth==1.14.3 +google-auth==1.15.0 google-cloud-bigquery-storage==0.8.0 google-cloud-bigquery==1.24.0 pyarrow==0.17.1 From 4daaff593ea040597edb31bf041891697a048907 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Fri, 29 May 2020 00:27:36 +0200 Subject: [PATCH 21/42] chore(deps): update dependency google-auth to v1.16.0 [(#3903)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/3903) --- samples/to_dataframe/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/to_dataframe/requirements.txt b/samples/to_dataframe/requirements.txt index 151995f8..bb7f7eb8 100644 --- a/samples/to_dataframe/requirements.txt +++ b/samples/to_dataframe/requirements.txt @@ -1,4 +1,4 @@ -google-auth==1.15.0 +google-auth==1.16.0 google-cloud-bigquery-storage==0.8.0 google-cloud-bigquery==1.24.0 pyarrow==0.17.1 From 3da28697cf7cc61c74a43b7120c5385719f69cb2 Mon Sep 17 00:00:00 2001 From: "Leah E. Cole" <6719667+leahecole@users.noreply.github.com> Date: Thu, 4 Jun 2020 14:34:05 -0700 Subject: [PATCH 22/42] update google-auth [(#3962)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/3962) --- samples/to_dataframe/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/to_dataframe/requirements.txt b/samples/to_dataframe/requirements.txt index bb7f7eb8..06d42322 100644 --- a/samples/to_dataframe/requirements.txt +++ b/samples/to_dataframe/requirements.txt @@ -1,4 +1,4 @@ -google-auth==1.16.0 +google-auth==1.16.1 google-cloud-bigquery-storage==0.8.0 google-cloud-bigquery==1.24.0 pyarrow==0.17.1 From 16b3b5664c1af60a97785e95de8afffa60487151 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Fri, 5 Jun 2020 01:34:33 +0200 Subject: [PATCH 23/42] Update dependency google-cloud-bigquery-storage to v1 [(#3968)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/3968) Co-authored-by: Leah E. Cole <6719667+leahecole@users.noreply.github.com> --- samples/to_dataframe/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/to_dataframe/requirements.txt b/samples/to_dataframe/requirements.txt index 06d42322..2f8614ad 100644 --- a/samples/to_dataframe/requirements.txt +++ b/samples/to_dataframe/requirements.txt @@ -1,5 +1,5 @@ google-auth==1.16.1 -google-cloud-bigquery-storage==0.8.0 +google-cloud-bigquery-storage==1.0.0 google-cloud-bigquery==1.24.0 pyarrow==0.17.1 ipython==7.10.2; python_version > '3.0' From 4e4726b18240ef64e9e96cd7405fe1c63590b85c Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Thu, 11 Jun 2020 06:36:11 +0200 Subject: [PATCH 24/42] Update dependency google-cloud-bigquery to v1.25.0 [(#4024)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/4024) This PR contains the following updates: | Package | Update | Change | |---|---|---| | [google-cloud-bigquery](https://togithub.com/googleapis/python-bigquery) | minor | `==1.24.0` -> `==1.25.0` | --- ### Release Notes
googleapis/python-bigquery ### [`v1.25.0`](https://togithub.com/googleapis/python-bigquery/blob/master/CHANGELOG.md#​1250-httpswwwgithubcomgoogleapispython-bigquerycomparev1240v1250-2020-06-06) [Compare Source](https://togithub.com/googleapis/python-bigquery/compare/v1.24.0...v1.25.0) ##### Features - add BigQuery storage client support to DB API ([#​36](https://www.github.com/googleapis/python-bigquery/issues/36)) ([ba9b2f8](https://www.github.com/googleapis/python-bigquery/commit/ba9b2f87e36320d80f6f6460b77e6daddb0fa214)) - **bigquery:** add create job method ([#​32](https://www.github.com/googleapis/python-bigquery/issues/32)) ([2abdef8](https://www.github.com/googleapis/python-bigquery/commit/2abdef82bed31601d1ca1aa92a10fea1e09f5297)) - **bigquery:** add support of model for extract job ([#​71](https://www.github.com/googleapis/python-bigquery/issues/71)) ([4a7a514](https://www.github.com/googleapis/python-bigquery/commit/4a7a514659a9f6f9bbd8af46bab3f8782d6b4b98)) - add HOUR support for time partitioning interval ([#​91](https://www.github.com/googleapis/python-bigquery/issues/91)) ([0dd90b9](https://www.github.com/googleapis/python-bigquery/commit/0dd90b90e3714c1d18f8a404917a9454870e338a)) - add support for policy tags ([#​77](https://www.github.com/googleapis/python-bigquery/issues/77)) ([38a5c01](https://www.github.com/googleapis/python-bigquery/commit/38a5c01ca830daf165592357c45f2fb4016aad23)) - make AccessEntry objects hashable ([#​93](https://www.github.com/googleapis/python-bigquery/issues/93)) ([23a173b](https://www.github.com/googleapis/python-bigquery/commit/23a173bc5a25c0c8200adc5af62eb05624c9099e)) - **bigquery:** expose start index parameter for query result ([#​121](https://www.github.com/googleapis/python-bigquery/issues/121)) ([be86de3](https://www.github.com/googleapis/python-bigquery/commit/be86de330a3c3801653a0ccef90e3d9bdb3eee7a)) - **bigquery:** unit and system test for dataframe with int column with Nan values ([#​39](https://www.github.com/googleapis/python-bigquery/issues/39)) ([5fd840e](https://www.github.com/googleapis/python-bigquery/commit/5fd840e9d4c592c4f736f2fd4792c9670ba6795e)) ##### Bug Fixes - allow partial streaming_buffer statistics ([#​37](https://www.github.com/googleapis/python-bigquery/issues/37)) ([645f0fd](https://www.github.com/googleapis/python-bigquery/commit/645f0fdb35ee0e81ee70f7459e796a42a1f03210)) - distinguish server timeouts from transport timeouts ([#​43](https://www.github.com/googleapis/python-bigquery/issues/43)) ([a17be5f](https://www.github.com/googleapis/python-bigquery/commit/a17be5f01043f32d9fbfb2ddf456031ea9205c8f)) - improve cell magic error message on missing query ([#​58](https://www.github.com/googleapis/python-bigquery/issues/58)) ([6182cf4](https://www.github.com/googleapis/python-bigquery/commit/6182cf48aef8f463bb96891cfc44a96768121dbc)) - **bigquery:** fix repr of model reference ([#​66](https://www.github.com/googleapis/python-bigquery/issues/66)) ([26c6204](https://www.github.com/googleapis/python-bigquery/commit/26c62046f4ec8880cf6561cc90a8b821dcc84ec5)) - **bigquery:** fix start index with page size for list rows ([#​27](https://www.github.com/googleapis/python-bigquery/issues/27)) ([400673b](https://www.github.com/googleapis/python-bigquery/commit/400673b5d0f2a6a3d828fdaad9d222ca967ffeff))
--- ### Renovate configuration :date: **Schedule**: At any time (no schedule defined). :vertical_traffic_light: **Automerge**: Disabled by config. Please merge this manually once you are satisfied. :recycle: **Rebasing**: Never, or you tick the rebase/retry checkbox. :no_bell: **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR has been generated by [WhiteSource Renovate](https://renovate.whitesourcesoftware.com). View repository job log [here](https://app.renovatebot.com/dashboard#GoogleCloudPlatform/python-docs-samples). --- samples/to_dataframe/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/to_dataframe/requirements.txt b/samples/to_dataframe/requirements.txt index 2f8614ad..d2273428 100644 --- a/samples/to_dataframe/requirements.txt +++ b/samples/to_dataframe/requirements.txt @@ -1,6 +1,6 @@ google-auth==1.16.1 google-cloud-bigquery-storage==1.0.0 -google-cloud-bigquery==1.24.0 +google-cloud-bigquery==1.25.0 pyarrow==0.17.1 ipython==7.10.2; python_version > '3.0' ipython==5.9.0; python_version < '3.0' From e9628e3f12fd5c990a3323aed12a4b7c549fc55d Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Thu, 11 Jun 2020 21:51:16 +0200 Subject: [PATCH 25/42] Update dependency google-auth to v1.17.0 [(#4058)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/4058) --- samples/to_dataframe/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/to_dataframe/requirements.txt b/samples/to_dataframe/requirements.txt index d2273428..bf8b3251 100644 --- a/samples/to_dataframe/requirements.txt +++ b/samples/to_dataframe/requirements.txt @@ -1,4 +1,4 @@ -google-auth==1.16.1 +google-auth==1.17.0 google-cloud-bigquery-storage==1.0.0 google-cloud-bigquery==1.25.0 pyarrow==0.17.1 From 20cde69290fd246c60a4374fd466c1d6f5730076 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Fri, 12 Jun 2020 02:32:11 +0200 Subject: [PATCH 26/42] chore(deps): update dependency google-auth to v1.17.1 [(#4073)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/4073) --- samples/to_dataframe/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/to_dataframe/requirements.txt b/samples/to_dataframe/requirements.txt index bf8b3251..d7ff3068 100644 --- a/samples/to_dataframe/requirements.txt +++ b/samples/to_dataframe/requirements.txt @@ -1,4 +1,4 @@ -google-auth==1.17.0 +google-auth==1.17.1 google-cloud-bigquery-storage==1.0.0 google-cloud-bigquery==1.25.0 pyarrow==0.17.1 From 5a96253f0b2b9d7038cb709b9d9fbfccc1efd0ef Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Fri, 12 Jun 2020 22:53:46 +0200 Subject: [PATCH 27/42] Update dependency google-auth to v1.17.2 [(#4083)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/4083) --- samples/to_dataframe/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/to_dataframe/requirements.txt b/samples/to_dataframe/requirements.txt index d7ff3068..a374f62a 100644 --- a/samples/to_dataframe/requirements.txt +++ b/samples/to_dataframe/requirements.txt @@ -1,4 +1,4 @@ -google-auth==1.17.1 +google-auth==1.17.2 google-cloud-bigquery-storage==1.0.0 google-cloud-bigquery==1.25.0 pyarrow==0.17.1 From ebaa1f837000a471f798f9075ff89691b4f778b8 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Fri, 19 Jun 2020 05:34:55 +0200 Subject: [PATCH 28/42] Update dependency google-auth to v1.18.0 [(#4125)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/4125) --- samples/to_dataframe/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/to_dataframe/requirements.txt b/samples/to_dataframe/requirements.txt index a374f62a..1edd61d9 100644 --- a/samples/to_dataframe/requirements.txt +++ b/samples/to_dataframe/requirements.txt @@ -1,4 +1,4 @@ -google-auth==1.17.2 +google-auth==1.18.0 google-cloud-bigquery-storage==1.0.0 google-cloud-bigquery==1.25.0 pyarrow==0.17.1 From 1f1e1602fbde0afbccbe636999a1a4f2ed6af870 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Mon, 13 Jul 2020 00:46:30 +0200 Subject: [PATCH 29/42] chore(deps): update dependency pytest to v5.4.3 [(#4279)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/4279) * chore(deps): update dependency pytest to v5.4.3 * specify pytest for python 2 in appengine Co-authored-by: Leah Cole --- samples/to_dataframe/requirements-test.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/to_dataframe/requirements-test.txt b/samples/to_dataframe/requirements-test.txt index 781d4326..79738af5 100644 --- a/samples/to_dataframe/requirements-test.txt +++ b/samples/to_dataframe/requirements-test.txt @@ -1 +1 @@ -pytest==5.3.2 +pytest==5.4.3 From 5069bee03957a43499c8751cebc7cd44751c8bb9 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Mon, 13 Jul 2020 22:20:34 +0200 Subject: [PATCH 30/42] chore(deps): update dependency google-auth to v1.19.0 [(#4293)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/4293) --- samples/to_dataframe/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/to_dataframe/requirements.txt b/samples/to_dataframe/requirements.txt index 1edd61d9..59f48f9d 100644 --- a/samples/to_dataframe/requirements.txt +++ b/samples/to_dataframe/requirements.txt @@ -1,4 +1,4 @@ -google-auth==1.18.0 +google-auth==1.19.0 google-cloud-bigquery-storage==1.0.0 google-cloud-bigquery==1.25.0 pyarrow==0.17.1 From 53b7efd8d0963ddca6a3cce219a73b315bc01131 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Fri, 17 Jul 2020 19:02:17 +0200 Subject: [PATCH 31/42] chore(deps): update dependency google-auth to v1.19.1 [(#4304)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/4304) --- samples/to_dataframe/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/to_dataframe/requirements.txt b/samples/to_dataframe/requirements.txt index 59f48f9d..d564fbdd 100644 --- a/samples/to_dataframe/requirements.txt +++ b/samples/to_dataframe/requirements.txt @@ -1,4 +1,4 @@ -google-auth==1.19.0 +google-auth==1.19.1 google-cloud-bigquery-storage==1.0.0 google-cloud-bigquery==1.25.0 pyarrow==0.17.1 From a835c2f3ff06b3b56bee71a0acec26fc5268ad58 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Sat, 18 Jul 2020 02:48:10 +0200 Subject: [PATCH 32/42] chore(deps): update dependency google-auth to v1.19.2 [(#4321)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/4321) This PR contains the following updates: | Package | Update | Change | |---|---|---| | [google-auth](https://togithub.com/googleapis/google-auth-library-python) | patch | `==1.19.1` -> `==1.19.2` | --- ### Release Notes
googleapis/google-auth-library-python ### [`v1.19.2`](https://togithub.com/googleapis/google-auth-library-python/blob/master/CHANGELOG.md#​1192-httpswwwgithubcomgoogleapisgoogle-auth-library-pythoncomparev1191v1192-2020-07-17) [Compare Source](https://togithub.com/googleapis/google-auth-library-python/compare/v1.19.1...v1.19.2)
--- ### Renovate configuration :date: **Schedule**: At any time (no schedule defined). :vertical_traffic_light: **Automerge**: Disabled by config. Please merge this manually once you are satisfied. :recycle: **Rebasing**: Never, or you tick the rebase/retry checkbox. :no_bell: **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR has been generated by [WhiteSource Renovate](https://renovate.whitesourcesoftware.com). View repository job log [here](https://app.renovatebot.com/dashboard#GoogleCloudPlatform/python-docs-samples). --- samples/to_dataframe/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/to_dataframe/requirements.txt b/samples/to_dataframe/requirements.txt index d564fbdd..0ea81c43 100644 --- a/samples/to_dataframe/requirements.txt +++ b/samples/to_dataframe/requirements.txt @@ -1,4 +1,4 @@ -google-auth==1.19.1 +google-auth==1.19.2 google-cloud-bigquery-storage==1.0.0 google-cloud-bigquery==1.25.0 pyarrow==0.17.1 From bfbdbd8c11214c3b4f81d5a7c71f26aa7d86481f Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Fri, 24 Jul 2020 19:24:14 +0200 Subject: [PATCH 33/42] chore(deps): update dependency pyarrow to v1 [(#4370)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/4370) --- samples/to_dataframe/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/to_dataframe/requirements.txt b/samples/to_dataframe/requirements.txt index 0ea81c43..a8c899e5 100644 --- a/samples/to_dataframe/requirements.txt +++ b/samples/to_dataframe/requirements.txt @@ -1,7 +1,7 @@ google-auth==1.19.2 google-cloud-bigquery-storage==1.0.0 google-cloud-bigquery==1.25.0 -pyarrow==0.17.1 +pyarrow==1.0.0 ipython==7.10.2; python_version > '3.0' ipython==5.9.0; python_version < '3.0' pandas==0.25.3; python_version > '3.0' From 6c86759464c2ba9d9f75fe8fb98ed7db1c00b0bc Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Tue, 28 Jul 2020 22:36:14 +0200 Subject: [PATCH 34/42] Update dependency google-auth to v1.20.0 [(#4387)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/4387) --- samples/to_dataframe/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/to_dataframe/requirements.txt b/samples/to_dataframe/requirements.txt index a8c899e5..d855f0fd 100644 --- a/samples/to_dataframe/requirements.txt +++ b/samples/to_dataframe/requirements.txt @@ -1,4 +1,4 @@ -google-auth==1.19.2 +google-auth==1.20.0 google-cloud-bigquery-storage==1.0.0 google-cloud-bigquery==1.25.0 pyarrow==1.0.0 From 208fe3de6663d808ae8d7ce516d88b490373c010 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Sat, 1 Aug 2020 21:51:00 +0200 Subject: [PATCH 35/42] Update dependency pytest to v6 [(#4390)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/4390) --- samples/to_dataframe/requirements-test.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/to_dataframe/requirements-test.txt b/samples/to_dataframe/requirements-test.txt index 79738af5..7e460c8c 100644 --- a/samples/to_dataframe/requirements-test.txt +++ b/samples/to_dataframe/requirements-test.txt @@ -1 +1 @@ -pytest==5.4.3 +pytest==6.0.1 From 1dc8306e66fef327a13687581dbe1ae1eb199684 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Fri, 7 Aug 2020 03:36:31 +0200 Subject: [PATCH 36/42] chore(deps): update dependency google-auth to v1.20.1 [(#4452)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/4452) --- samples/to_dataframe/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/to_dataframe/requirements.txt b/samples/to_dataframe/requirements.txt index d855f0fd..80990849 100644 --- a/samples/to_dataframe/requirements.txt +++ b/samples/to_dataframe/requirements.txt @@ -1,4 +1,4 @@ -google-auth==1.20.0 +google-auth==1.20.1 google-cloud-bigquery-storage==1.0.0 google-cloud-bigquery==1.25.0 pyarrow==1.0.0 From 07c89a34e10b1ca6c083c4e204b5fdae902444d1 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Thu, 27 Aug 2020 05:43:45 +0200 Subject: [PATCH 37/42] chore(deps): update dependency pyarrow to v1.0.1 [(#4566)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/4566) --- samples/to_dataframe/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/to_dataframe/requirements.txt b/samples/to_dataframe/requirements.txt index 80990849..6440b10d 100644 --- a/samples/to_dataframe/requirements.txt +++ b/samples/to_dataframe/requirements.txt @@ -1,7 +1,7 @@ google-auth==1.20.1 google-cloud-bigquery-storage==1.0.0 google-cloud-bigquery==1.25.0 -pyarrow==1.0.0 +pyarrow==1.0.1 ipython==7.10.2; python_version > '3.0' ipython==5.9.0; python_version < '3.0' pandas==0.25.3; python_version > '3.0' From db828495dab260dc22a0bf0d64a5c32874d962a5 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Fri, 28 Aug 2020 01:17:31 +0200 Subject: [PATCH 38/42] Update dependency google-auth to v1.21.0 [(#4588)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/4588) --- samples/to_dataframe/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/to_dataframe/requirements.txt b/samples/to_dataframe/requirements.txt index 6440b10d..b5ac6177 100644 --- a/samples/to_dataframe/requirements.txt +++ b/samples/to_dataframe/requirements.txt @@ -1,4 +1,4 @@ -google-auth==1.20.1 +google-auth==1.21.0 google-cloud-bigquery-storage==1.0.0 google-cloud-bigquery==1.25.0 pyarrow==1.0.1 From b4deb53f2aec2893aa9fe5dc2b77308a17565707 Mon Sep 17 00:00:00 2001 From: Peter Lamut Date: Wed, 2 Sep 2020 16:03:50 +0200 Subject: [PATCH 39/42] chore: update templates --- .github/CODEOWNERS | 8 + .github/snippet-bot.yml | 0 .gitignore | 3 +- .kokoro/build.sh | 8 +- .kokoro/docker/docs/Dockerfile | 98 ++++++ .kokoro/docker/docs/fetch_gpg_keys.sh | 45 +++ .kokoro/docs/common.cfg | 21 +- .kokoro/docs/docs-presubmit.cfg | 17 + .kokoro/publish-docs.sh | 39 ++- .kokoro/trampoline_v2.sh | 487 ++++++++++++++++++++++++++ .trampolinerc | 51 +++ docs/conf.py | 11 +- noxfile.py | 37 ++ samples/AUTHORING_GUIDE.md | 1 + samples/CONTRIBUTING.md | 1 + samples/noxfile.py | 222 ++++++++++++ samples/to_dataframe/main_test.py | 9 +- samples/to_dataframe/noxfile.py | 222 ++++++++++++ synth.metadata | 17 +- synth.py | 10 + 20 files changed, 1275 insertions(+), 32 deletions(-) create mode 100644 .github/CODEOWNERS create mode 100644 .github/snippet-bot.yml create mode 100644 .kokoro/docker/docs/Dockerfile create mode 100755 .kokoro/docker/docs/fetch_gpg_keys.sh create mode 100644 .kokoro/docs/docs-presubmit.cfg create mode 100755 .kokoro/trampoline_v2.sh create mode 100644 .trampolinerc create mode 100644 samples/AUTHORING_GUIDE.md create mode 100644 samples/CONTRIBUTING.md create mode 100644 samples/noxfile.py create mode 100644 samples/to_dataframe/noxfile.py diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 00000000..22178c17 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,8 @@ +# Code owners file. +# This file controls who is tagged for review for any given pull request. +# +# For syntax help see: +# https://help.github.com/en/github/creating-cloning-and-archiving-repositories/about-code-owners#codeowners-syntax + + +/samples/ @shollyman @googleapis/python-samples-owners diff --git a/.github/snippet-bot.yml b/.github/snippet-bot.yml new file mode 100644 index 00000000..e69de29b diff --git a/.gitignore b/.gitignore index b87e1ed5..b9daa52f 100644 --- a/.gitignore +++ b/.gitignore @@ -46,6 +46,7 @@ pip-log.txt # Built documentation docs/_build bigquery/docs/generated +docs.metadata # Virtual environment env/ @@ -57,4 +58,4 @@ system_tests/local_test_setup # Make sure a generated file isn't accidentally committed. pylintrc -pylintrc.test \ No newline at end of file +pylintrc.test diff --git a/.kokoro/build.sh b/.kokoro/build.sh index e147fdfe..a7968df7 100755 --- a/.kokoro/build.sh +++ b/.kokoro/build.sh @@ -36,4 +36,10 @@ python3.6 -m pip uninstall --yes --quiet nox-automation python3.6 -m pip install --upgrade --quiet nox python3.6 -m nox --version -python3.6 -m nox +# If NOX_SESSION is set, it only runs the specified session, +# otherwise run all the sessions. +if [[ -n "${NOX_SESSION:-}" ]]; then + python3.6 -m nox -s "${NOX_SESSION:-}" +else + python3.6 -m nox +fi diff --git a/.kokoro/docker/docs/Dockerfile b/.kokoro/docker/docs/Dockerfile new file mode 100644 index 00000000..412b0b56 --- /dev/null +++ b/.kokoro/docker/docs/Dockerfile @@ -0,0 +1,98 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ubuntu:20.04 + +ENV DEBIAN_FRONTEND noninteractive + +# Ensure local Python is preferred over distribution Python. +ENV PATH /usr/local/bin:$PATH + +# Install dependencies. +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + apt-transport-https \ + build-essential \ + ca-certificates \ + curl \ + dirmngr \ + git \ + gpg-agent \ + graphviz \ + libbz2-dev \ + libdb5.3-dev \ + libexpat1-dev \ + libffi-dev \ + liblzma-dev \ + libreadline-dev \ + libsnappy-dev \ + libssl-dev \ + libsqlite3-dev \ + portaudio19-dev \ + redis-server \ + software-properties-common \ + ssh \ + sudo \ + tcl \ + tcl-dev \ + tk \ + tk-dev \ + uuid-dev \ + wget \ + zlib1g-dev \ + && add-apt-repository universe \ + && apt-get update \ + && apt-get -y install jq \ + && apt-get clean autoclean \ + && apt-get autoremove -y \ + && rm -rf /var/lib/apt/lists/* \ + && rm -f /var/cache/apt/archives/*.deb + + +COPY fetch_gpg_keys.sh /tmp +# Install the desired versions of Python. +RUN set -ex \ + && export GNUPGHOME="$(mktemp -d)" \ + && echo "disable-ipv6" >> "${GNUPGHOME}/dirmngr.conf" \ + && /tmp/fetch_gpg_keys.sh \ + && for PYTHON_VERSION in 3.7.8 3.8.5; do \ + wget --no-check-certificate -O python-${PYTHON_VERSION}.tar.xz "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz" \ + && wget --no-check-certificate -O python-${PYTHON_VERSION}.tar.xz.asc "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc" \ + && gpg --batch --verify python-${PYTHON_VERSION}.tar.xz.asc python-${PYTHON_VERSION}.tar.xz \ + && rm -r python-${PYTHON_VERSION}.tar.xz.asc \ + && mkdir -p /usr/src/python-${PYTHON_VERSION} \ + && tar -xJC /usr/src/python-${PYTHON_VERSION} --strip-components=1 -f python-${PYTHON_VERSION}.tar.xz \ + && rm python-${PYTHON_VERSION}.tar.xz \ + && cd /usr/src/python-${PYTHON_VERSION} \ + && ./configure \ + --enable-shared \ + # This works only on Python 2.7 and throws a warning on every other + # version, but seems otherwise harmless. + --enable-unicode=ucs4 \ + --with-system-ffi \ + --without-ensurepip \ + && make -j$(nproc) \ + && make install \ + && ldconfig \ + ; done \ + && rm -rf "${GNUPGHOME}" \ + && rm -rf /usr/src/python* \ + && rm -rf ~/.cache/ + +RUN wget -O /tmp/get-pip.py 'https://bootstrap.pypa.io/get-pip.py' \ + && python3.7 /tmp/get-pip.py \ + && python3.8 /tmp/get-pip.py \ + && rm /tmp/get-pip.py + +CMD ["python3.7"] diff --git a/.kokoro/docker/docs/fetch_gpg_keys.sh b/.kokoro/docker/docs/fetch_gpg_keys.sh new file mode 100755 index 00000000..d653dd86 --- /dev/null +++ b/.kokoro/docker/docs/fetch_gpg_keys.sh @@ -0,0 +1,45 @@ +#!/bin/bash +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# A script to fetch gpg keys with retry. +# Avoid jinja parsing the file. +# + +function retry { + if [[ "${#}" -le 1 ]]; then + echo "Usage: ${0} retry_count commands.." + exit 1 + fi + local retries=${1} + local command="${@:2}" + until [[ "${retries}" -le 0 ]]; do + $command && return 0 + if [[ $? -ne 0 ]]; then + echo "command failed, retrying" + ((retries--)) + fi + done + return 1 +} + +# 3.6.9, 3.7.5 (Ned Deily) +retry 3 gpg --keyserver ha.pool.sks-keyservers.net --recv-keys \ + 0D96DF4D4110E5C43FBFB17F2D347EA6AA65421D + +# 3.8.0 (Łukasz Langa) +retry 3 gpg --keyserver ha.pool.sks-keyservers.net --recv-keys \ + E3FF2839C048B25C084DEBE9B26995E310250568 + +# diff --git a/.kokoro/docs/common.cfg b/.kokoro/docs/common.cfg index 715759c8..22832836 100644 --- a/.kokoro/docs/common.cfg +++ b/.kokoro/docs/common.cfg @@ -11,12 +11,12 @@ action { gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" # Use the trampoline script to run in docker. -build_file: "python-bigquery-storage/.kokoro/trampoline.sh" +build_file: "python-bigquery-storage/.kokoro/trampoline_v2.sh" # Configure the docker image for kokoro-trampoline. env_vars: { key: "TRAMPOLINE_IMAGE" - value: "gcr.io/cloud-devrel-kokoro-resources/python-multi" + value: "gcr.io/cloud-devrel-kokoro-resources/python-lib-docs" } env_vars: { key: "TRAMPOLINE_BUILD_FILE" @@ -28,6 +28,23 @@ env_vars: { value: "docs-staging" } +env_vars: { + key: "V2_STAGING_BUCKET" + value: "docs-staging-v2-staging" +} + +# It will upload the docker image after successful builds. +env_vars: { + key: "TRAMPOLINE_IMAGE_UPLOAD" + value: "true" +} + +# It will always build the docker image. +env_vars: { + key: "TRAMPOLINE_DOCKERFILE" + value: ".kokoro/docker/docs/Dockerfile" +} + # Fetch the token needed for reporting release status to GitHub before_action { fetch_keystore { diff --git a/.kokoro/docs/docs-presubmit.cfg b/.kokoro/docs/docs-presubmit.cfg new file mode 100644 index 00000000..11181078 --- /dev/null +++ b/.kokoro/docs/docs-presubmit.cfg @@ -0,0 +1,17 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "STAGING_BUCKET" + value: "gcloud-python-test" +} + +env_vars: { + key: "V2_STAGING_BUCKET" + value: "gcloud-python-test" +} + +# We only upload the image in the main `docs` build. +env_vars: { + key: "TRAMPOLINE_IMAGE_UPLOAD" + value: "false" +} diff --git a/.kokoro/publish-docs.sh b/.kokoro/publish-docs.sh index 06d2d9b8..8acb14e8 100755 --- a/.kokoro/publish-docs.sh +++ b/.kokoro/publish-docs.sh @@ -18,26 +18,16 @@ set -eo pipefail # Disable buffering, so that the logs stream through. export PYTHONUNBUFFERED=1 -cd github/python-bigquery-storage - -# Remove old nox -python3.6 -m pip uninstall --yes --quiet nox-automation +export PATH="${HOME}/.local/bin:${PATH}" # Install nox -python3.6 -m pip install --upgrade --quiet nox -python3.6 -m nox --version +python3 -m pip install --user --upgrade --quiet nox +python3 -m nox --version # build docs nox -s docs -python3 -m pip install gcp-docuploader - -# install a json parser -sudo apt-get update -sudo apt-get -y install software-properties-common -sudo add-apt-repository universe -sudo apt-get update -sudo apt-get -y install jq +python3 -m pip install --user gcp-docuploader # create metadata python3 -m docuploader create-metadata \ @@ -52,4 +42,23 @@ python3 -m docuploader create-metadata \ cat docs.metadata # upload docs -python3 -m docuploader upload docs/_build/html --metadata-file docs.metadata --staging-bucket docs-staging +python3 -m docuploader upload docs/_build/html --metadata-file docs.metadata --staging-bucket "${STAGING_BUCKET}" + + +# docfx yaml files +nox -s docfx + +# create metadata. +python3 -m docuploader create-metadata \ + --name=$(jq --raw-output '.name // empty' .repo-metadata.json) \ + --version=$(python3 setup.py --version) \ + --language=$(jq --raw-output '.language // empty' .repo-metadata.json) \ + --distribution-name=$(python3 setup.py --name) \ + --product-page=$(jq --raw-output '.product_documentation // empty' .repo-metadata.json) \ + --github-repository=$(jq --raw-output '.repo // empty' .repo-metadata.json) \ + --issue-tracker=$(jq --raw-output '.issue_tracker // empty' .repo-metadata.json) + +cat docs.metadata + +# upload docs +python3 -m docuploader upload docs/_build/html/docfx_yaml --metadata-file docs.metadata --destination-prefix docfx --staging-bucket "${V2_STAGING_BUCKET}" diff --git a/.kokoro/trampoline_v2.sh b/.kokoro/trampoline_v2.sh new file mode 100755 index 00000000..719bcd5b --- /dev/null +++ b/.kokoro/trampoline_v2.sh @@ -0,0 +1,487 @@ +#!/usr/bin/env bash +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# trampoline_v2.sh +# +# This script does 3 things. +# +# 1. Prepare the Docker image for the test +# 2. Run the Docker with appropriate flags to run the test +# 3. Upload the newly built Docker image +# +# in a way that is somewhat compatible with trampoline_v1. +# +# To run this script, first download few files from gcs to /dev/shm. +# (/dev/shm is passed into the container as KOKORO_GFILE_DIR). +# +# gsutil cp gs://cloud-devrel-kokoro-resources/python-docs-samples/secrets_viewer_service_account.json /dev/shm +# gsutil cp gs://cloud-devrel-kokoro-resources/python-docs-samples/automl_secrets.txt /dev/shm +# +# Then run the script. +# .kokoro/trampoline_v2.sh +# +# These environment variables are required: +# TRAMPOLINE_IMAGE: The docker image to use. +# TRAMPOLINE_DOCKERFILE: The location of the Dockerfile. +# +# You can optionally change these environment variables: +# TRAMPOLINE_IMAGE_UPLOAD: +# (true|false): Whether to upload the Docker image after the +# successful builds. +# TRAMPOLINE_BUILD_FILE: The script to run in the docker container. +# TRAMPOLINE_WORKSPACE: The workspace path in the docker container. +# Defaults to /workspace. +# Potentially there are some repo specific envvars in .trampolinerc in +# the project root. + + +set -euo pipefail + +TRAMPOLINE_VERSION="2.0.5" + +if command -v tput >/dev/null && [[ -n "${TERM:-}" ]]; then + readonly IO_COLOR_RED="$(tput setaf 1)" + readonly IO_COLOR_GREEN="$(tput setaf 2)" + readonly IO_COLOR_YELLOW="$(tput setaf 3)" + readonly IO_COLOR_RESET="$(tput sgr0)" +else + readonly IO_COLOR_RED="" + readonly IO_COLOR_GREEN="" + readonly IO_COLOR_YELLOW="" + readonly IO_COLOR_RESET="" +fi + +function function_exists { + [ $(LC_ALL=C type -t $1)"" == "function" ] +} + +# Logs a message using the given color. The first argument must be one +# of the IO_COLOR_* variables defined above, such as +# "${IO_COLOR_YELLOW}". The remaining arguments will be logged in the +# given color. The log message will also have an RFC-3339 timestamp +# prepended (in UTC). You can disable the color output by setting +# TERM=vt100. +function log_impl() { + local color="$1" + shift + local timestamp="$(date -u "+%Y-%m-%dT%H:%M:%SZ")" + echo "================================================================" + echo "${color}${timestamp}:" "$@" "${IO_COLOR_RESET}" + echo "================================================================" +} + +# Logs the given message with normal coloring and a timestamp. +function log() { + log_impl "${IO_COLOR_RESET}" "$@" +} + +# Logs the given message in green with a timestamp. +function log_green() { + log_impl "${IO_COLOR_GREEN}" "$@" +} + +# Logs the given message in yellow with a timestamp. +function log_yellow() { + log_impl "${IO_COLOR_YELLOW}" "$@" +} + +# Logs the given message in red with a timestamp. +function log_red() { + log_impl "${IO_COLOR_RED}" "$@" +} + +readonly tmpdir=$(mktemp -d -t ci-XXXXXXXX) +readonly tmphome="${tmpdir}/h" +mkdir -p "${tmphome}" + +function cleanup() { + rm -rf "${tmpdir}" +} +trap cleanup EXIT + +RUNNING_IN_CI="${RUNNING_IN_CI:-false}" + +# The workspace in the container, defaults to /workspace. +TRAMPOLINE_WORKSPACE="${TRAMPOLINE_WORKSPACE:-/workspace}" + +pass_down_envvars=( + # TRAMPOLINE_V2 variables. + # Tells scripts whether they are running as part of CI or not. + "RUNNING_IN_CI" + # Indicates which CI system we're in. + "TRAMPOLINE_CI" + # Indicates the version of the script. + "TRAMPOLINE_VERSION" +) + +log_yellow "Building with Trampoline ${TRAMPOLINE_VERSION}" + +# Detect which CI systems we're in. If we're in any of the CI systems +# we support, `RUNNING_IN_CI` will be true and `TRAMPOLINE_CI` will be +# the name of the CI system. Both envvars will be passing down to the +# container for telling which CI system we're in. +if [[ -n "${KOKORO_BUILD_ID:-}" ]]; then + # descriptive env var for indicating it's on CI. + RUNNING_IN_CI="true" + TRAMPOLINE_CI="kokoro" + if [[ "${TRAMPOLINE_USE_LEGACY_SERVICE_ACCOUNT:-}" == "true" ]]; then + if [[ ! -f "${KOKORO_GFILE_DIR}/kokoro-trampoline.service-account.json" ]]; then + log_red "${KOKORO_GFILE_DIR}/kokoro-trampoline.service-account.json does not exist. Did you forget to mount cloud-devrel-kokoro-resources/trampoline? Aborting." + exit 1 + fi + # This service account will be activated later. + TRAMPOLINE_SERVICE_ACCOUNT="${KOKORO_GFILE_DIR}/kokoro-trampoline.service-account.json" + else + if [[ "${TRAMPOLINE_VERBOSE:-}" == "true" ]]; then + gcloud auth list + fi + log_yellow "Configuring Container Registry access" + gcloud auth configure-docker --quiet + fi + pass_down_envvars+=( + # KOKORO dynamic variables. + "KOKORO_BUILD_NUMBER" + "KOKORO_BUILD_ID" + "KOKORO_JOB_NAME" + "KOKORO_GIT_COMMIT" + "KOKORO_GITHUB_COMMIT" + "KOKORO_GITHUB_PULL_REQUEST_NUMBER" + "KOKORO_GITHUB_PULL_REQUEST_COMMIT" + # For Build Cop Bot + "KOKORO_GITHUB_COMMIT_URL" + "KOKORO_GITHUB_PULL_REQUEST_URL" + ) +elif [[ "${TRAVIS:-}" == "true" ]]; then + RUNNING_IN_CI="true" + TRAMPOLINE_CI="travis" + pass_down_envvars+=( + "TRAVIS_BRANCH" + "TRAVIS_BUILD_ID" + "TRAVIS_BUILD_NUMBER" + "TRAVIS_BUILD_WEB_URL" + "TRAVIS_COMMIT" + "TRAVIS_COMMIT_MESSAGE" + "TRAVIS_COMMIT_RANGE" + "TRAVIS_JOB_NAME" + "TRAVIS_JOB_NUMBER" + "TRAVIS_JOB_WEB_URL" + "TRAVIS_PULL_REQUEST" + "TRAVIS_PULL_REQUEST_BRANCH" + "TRAVIS_PULL_REQUEST_SHA" + "TRAVIS_PULL_REQUEST_SLUG" + "TRAVIS_REPO_SLUG" + "TRAVIS_SECURE_ENV_VARS" + "TRAVIS_TAG" + ) +elif [[ -n "${GITHUB_RUN_ID:-}" ]]; then + RUNNING_IN_CI="true" + TRAMPOLINE_CI="github-workflow" + pass_down_envvars+=( + "GITHUB_WORKFLOW" + "GITHUB_RUN_ID" + "GITHUB_RUN_NUMBER" + "GITHUB_ACTION" + "GITHUB_ACTIONS" + "GITHUB_ACTOR" + "GITHUB_REPOSITORY" + "GITHUB_EVENT_NAME" + "GITHUB_EVENT_PATH" + "GITHUB_SHA" + "GITHUB_REF" + "GITHUB_HEAD_REF" + "GITHUB_BASE_REF" + ) +elif [[ "${CIRCLECI:-}" == "true" ]]; then + RUNNING_IN_CI="true" + TRAMPOLINE_CI="circleci" + pass_down_envvars+=( + "CIRCLE_BRANCH" + "CIRCLE_BUILD_NUM" + "CIRCLE_BUILD_URL" + "CIRCLE_COMPARE_URL" + "CIRCLE_JOB" + "CIRCLE_NODE_INDEX" + "CIRCLE_NODE_TOTAL" + "CIRCLE_PREVIOUS_BUILD_NUM" + "CIRCLE_PROJECT_REPONAME" + "CIRCLE_PROJECT_USERNAME" + "CIRCLE_REPOSITORY_URL" + "CIRCLE_SHA1" + "CIRCLE_STAGE" + "CIRCLE_USERNAME" + "CIRCLE_WORKFLOW_ID" + "CIRCLE_WORKFLOW_JOB_ID" + "CIRCLE_WORKFLOW_UPSTREAM_JOB_IDS" + "CIRCLE_WORKFLOW_WORKSPACE_ID" + ) +fi + +# Configure the service account for pulling the docker image. +function repo_root() { + local dir="$1" + while [[ ! -d "${dir}/.git" ]]; do + dir="$(dirname "$dir")" + done + echo "${dir}" +} + +# Detect the project root. In CI builds, we assume the script is in +# the git tree and traverse from there, otherwise, traverse from `pwd` +# to find `.git` directory. +if [[ "${RUNNING_IN_CI:-}" == "true" ]]; then + PROGRAM_PATH="$(realpath "$0")" + PROGRAM_DIR="$(dirname "${PROGRAM_PATH}")" + PROJECT_ROOT="$(repo_root "${PROGRAM_DIR}")" +else + PROJECT_ROOT="$(repo_root $(pwd))" +fi + +log_yellow "Changing to the project root: ${PROJECT_ROOT}." +cd "${PROJECT_ROOT}" + +# To support relative path for `TRAMPOLINE_SERVICE_ACCOUNT`, we need +# to use this environment variable in `PROJECT_ROOT`. +if [[ -n "${TRAMPOLINE_SERVICE_ACCOUNT:-}" ]]; then + + mkdir -p "${tmpdir}/gcloud" + gcloud_config_dir="${tmpdir}/gcloud" + + log_yellow "Using isolated gcloud config: ${gcloud_config_dir}." + export CLOUDSDK_CONFIG="${gcloud_config_dir}" + + log_yellow "Using ${TRAMPOLINE_SERVICE_ACCOUNT} for authentication." + gcloud auth activate-service-account \ + --key-file "${TRAMPOLINE_SERVICE_ACCOUNT}" + log_yellow "Configuring Container Registry access" + gcloud auth configure-docker --quiet +fi + +required_envvars=( + # The basic trampoline configurations. + "TRAMPOLINE_IMAGE" + "TRAMPOLINE_BUILD_FILE" +) + +if [[ -f "${PROJECT_ROOT}/.trampolinerc" ]]; then + source "${PROJECT_ROOT}/.trampolinerc" +fi + +log_yellow "Checking environment variables." +for e in "${required_envvars[@]}" +do + if [[ -z "${!e:-}" ]]; then + log "Missing ${e} env var. Aborting." + exit 1 + fi +done + +# We want to support legacy style TRAMPOLINE_BUILD_FILE used with V1 +# script: e.g. "github/repo-name/.kokoro/run_tests.sh" +TRAMPOLINE_BUILD_FILE="${TRAMPOLINE_BUILD_FILE#github/*/}" +log_yellow "Using TRAMPOLINE_BUILD_FILE: ${TRAMPOLINE_BUILD_FILE}" + +# ignore error on docker operations and test execution +set +e + +log_yellow "Preparing Docker image." +# We only download the docker image in CI builds. +if [[ "${RUNNING_IN_CI:-}" == "true" ]]; then + # Download the docker image specified by `TRAMPOLINE_IMAGE` + + # We may want to add --max-concurrent-downloads flag. + + log_yellow "Start pulling the Docker image: ${TRAMPOLINE_IMAGE}." + if docker pull "${TRAMPOLINE_IMAGE}"; then + log_green "Finished pulling the Docker image: ${TRAMPOLINE_IMAGE}." + has_image="true" + else + log_red "Failed pulling the Docker image: ${TRAMPOLINE_IMAGE}." + has_image="false" + fi +else + # For local run, check if we have the image. + if docker images "${TRAMPOLINE_IMAGE}:latest" | grep "${TRAMPOLINE_IMAGE}"; then + has_image="true" + else + has_image="false" + fi +fi + + +# The default user for a Docker container has uid 0 (root). To avoid +# creating root-owned files in the build directory we tell docker to +# use the current user ID. +user_uid="$(id -u)" +user_gid="$(id -g)" +user_name="$(id -un)" + +# To allow docker in docker, we add the user to the docker group in +# the host os. +docker_gid=$(cut -d: -f3 < <(getent group docker)) + +update_cache="false" +if [[ "${TRAMPOLINE_DOCKERFILE:-none}" != "none" ]]; then + # Build the Docker image from the source. + context_dir=$(dirname "${TRAMPOLINE_DOCKERFILE}") + docker_build_flags=( + "-f" "${TRAMPOLINE_DOCKERFILE}" + "-t" "${TRAMPOLINE_IMAGE}" + "--build-arg" "UID=${user_uid}" + "--build-arg" "USERNAME=${user_name}" + ) + if [[ "${has_image}" == "true" ]]; then + docker_build_flags+=("--cache-from" "${TRAMPOLINE_IMAGE}") + fi + + log_yellow "Start building the docker image." + if [[ "${TRAMPOLINE_VERBOSE:-false}" == "true" ]]; then + echo "docker build" "${docker_build_flags[@]}" "${context_dir}" + fi + + # ON CI systems, we want to suppress docker build logs, only + # output the logs when it fails. + if [[ "${RUNNING_IN_CI:-}" == "true" ]]; then + if docker build "${docker_build_flags[@]}" "${context_dir}" \ + > "${tmpdir}/docker_build.log" 2>&1; then + if [[ "${TRAMPOLINE_VERBOSE:-}" == "true" ]]; then + cat "${tmpdir}/docker_build.log" + fi + + log_green "Finished building the docker image." + update_cache="true" + else + log_red "Failed to build the Docker image, aborting." + log_yellow "Dumping the build logs:" + cat "${tmpdir}/docker_build.log" + exit 1 + fi + else + if docker build "${docker_build_flags[@]}" "${context_dir}"; then + log_green "Finished building the docker image." + update_cache="true" + else + log_red "Failed to build the Docker image, aborting." + exit 1 + fi + fi +else + if [[ "${has_image}" != "true" ]]; then + log_red "We do not have ${TRAMPOLINE_IMAGE} locally, aborting." + exit 1 + fi +fi + +# We use an array for the flags so they are easier to document. +docker_flags=( + # Remove the container after it exists. + "--rm" + + # Use the host network. + "--network=host" + + # Run in priviledged mode. We are not using docker for sandboxing or + # isolation, just for packaging our dev tools. + "--privileged" + + # Run the docker script with the user id. Because the docker image gets to + # write in ${PWD} you typically want this to be your user id. + # To allow docker in docker, we need to use docker gid on the host. + "--user" "${user_uid}:${docker_gid}" + + # Pass down the USER. + "--env" "USER=${user_name}" + + # Mount the project directory inside the Docker container. + "--volume" "${PROJECT_ROOT}:${TRAMPOLINE_WORKSPACE}" + "--workdir" "${TRAMPOLINE_WORKSPACE}" + "--env" "PROJECT_ROOT=${TRAMPOLINE_WORKSPACE}" + + # Mount the temporary home directory. + "--volume" "${tmphome}:/h" + "--env" "HOME=/h" + + # Allow docker in docker. + "--volume" "/var/run/docker.sock:/var/run/docker.sock" + + # Mount the /tmp so that docker in docker can mount the files + # there correctly. + "--volume" "/tmp:/tmp" + # Pass down the KOKORO_GFILE_DIR and KOKORO_KEYSTORE_DIR + # TODO(tmatsuo): This part is not portable. + "--env" "TRAMPOLINE_SECRET_DIR=/secrets" + "--volume" "${KOKORO_GFILE_DIR:-/dev/shm}:/secrets/gfile" + "--env" "KOKORO_GFILE_DIR=/secrets/gfile" + "--volume" "${KOKORO_KEYSTORE_DIR:-/dev/shm}:/secrets/keystore" + "--env" "KOKORO_KEYSTORE_DIR=/secrets/keystore" +) + +# Add an option for nicer output if the build gets a tty. +if [[ -t 0 ]]; then + docker_flags+=("-it") +fi + +# Passing down env vars +for e in "${pass_down_envvars[@]}" +do + if [[ -n "${!e:-}" ]]; then + docker_flags+=("--env" "${e}=${!e}") + fi +done + +# If arguments are given, all arguments will become the commands run +# in the container, otherwise run TRAMPOLINE_BUILD_FILE. +if [[ $# -ge 1 ]]; then + log_yellow "Running the given commands '" "${@:1}" "' in the container." + readonly commands=("${@:1}") + if [[ "${TRAMPOLINE_VERBOSE:-}" == "true" ]]; then + echo docker run "${docker_flags[@]}" "${TRAMPOLINE_IMAGE}" "${commands[@]}" + fi + docker run "${docker_flags[@]}" "${TRAMPOLINE_IMAGE}" "${commands[@]}" +else + log_yellow "Running the tests in a Docker container." + docker_flags+=("--entrypoint=${TRAMPOLINE_BUILD_FILE}") + if [[ "${TRAMPOLINE_VERBOSE:-}" == "true" ]]; then + echo docker run "${docker_flags[@]}" "${TRAMPOLINE_IMAGE}" + fi + docker run "${docker_flags[@]}" "${TRAMPOLINE_IMAGE}" +fi + + +test_retval=$? + +if [[ ${test_retval} -eq 0 ]]; then + log_green "Build finished with ${test_retval}" +else + log_red "Build finished with ${test_retval}" +fi + +# Only upload it when the test passes. +if [[ "${update_cache}" == "true" ]] && \ + [[ $test_retval == 0 ]] && \ + [[ "${TRAMPOLINE_IMAGE_UPLOAD:-false}" == "true" ]]; then + log_yellow "Uploading the Docker image." + if docker push "${TRAMPOLINE_IMAGE}"; then + log_green "Finished uploading the Docker image." + else + log_red "Failed uploading the Docker image." + fi + # Call trampoline_after_upload_hook if it's defined. + if function_exists trampoline_after_upload_hook; then + trampoline_after_upload_hook + fi + +fi + +exit "${test_retval}" diff --git a/.trampolinerc b/.trampolinerc new file mode 100644 index 00000000..995ee291 --- /dev/null +++ b/.trampolinerc @@ -0,0 +1,51 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Template for .trampolinerc + +# Add required env vars here. +required_envvars+=( + "STAGING_BUCKET" + "V2_STAGING_BUCKET" +) + +# Add env vars which are passed down into the container here. +pass_down_envvars+=( + "STAGING_BUCKET" + "V2_STAGING_BUCKET" +) + +# Prevent unintentional override on the default image. +if [[ "${TRAMPOLINE_IMAGE_UPLOAD:-false}" == "true" ]] && \ + [[ -z "${TRAMPOLINE_IMAGE:-}" ]]; then + echo "Please set TRAMPOLINE_IMAGE if you want to upload the Docker image." + exit 1 +fi + +# Define the default value if it makes sense. +if [[ -z "${TRAMPOLINE_IMAGE_UPLOAD:-}" ]]; then + TRAMPOLINE_IMAGE_UPLOAD="" +fi + +if [[ -z "${TRAMPOLINE_IMAGE:-}" ]]; then + TRAMPOLINE_IMAGE="" +fi + +if [[ -z "${TRAMPOLINE_DOCKERFILE:-}" ]]; then + TRAMPOLINE_DOCKERFILE="" +fi + +if [[ -z "${TRAMPOLINE_BUILD_FILE:-}" ]]; then + TRAMPOLINE_BUILD_FILE="" +fi diff --git a/docs/conf.py b/docs/conf.py index b331b183..905bd12e 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -20,6 +20,10 @@ # documentation root, use os.path.abspath to make it absolute, like shown here. sys.path.insert(0, os.path.abspath("..")) +# For plugins that can not read conf.py. +# See also: https://github.com/docascode/sphinx-docfx-yaml/issues/85 +sys.path.insert(0, os.path.abspath(".")) + __version__ = "" # -- General configuration ------------------------------------------------ @@ -90,7 +94,12 @@ # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. -exclude_patterns = ["_build"] +exclude_patterns = [ + "_build", + "samples/AUTHORING_GUIDE.md", + "samples/CONTRIBUTING.md", + "samples/snippets/README.rst", +] # The reST default role (used for this markup: `text`) to use for all # documents. diff --git a/noxfile.py b/noxfile.py index 716d6f03..e476c62c 100644 --- a/noxfile.py +++ b/noxfile.py @@ -101,6 +101,10 @@ def system(session): """Run the system test suite.""" system_test_path = os.path.join("tests", "system.py") system_test_folder_path = os.path.join("tests", "system") + + # Check the value of `RUN_SYSTEM_TESTS` env var. It defaults to true. + if os.environ.get("RUN_SYSTEM_TESTS", "true") == "false": + session.skip("RUN_SYSTEM_TESTS is set to false, skipping") # Sanity check: Only run tests if the environment variable is set. if not os.environ.get("GOOGLE_APPLICATION_CREDENTIALS", ""): session.skip("Credentials must be set via environment variable") @@ -181,3 +185,36 @@ def docs(session): os.path.join("docs", ""), os.path.join("docs", "_build", "html", ""), ) + + +@nox.session(python=DEFAULT_PYTHON_VERSION) +def docfx(session): + """Build the docfx yaml files for this library.""" + + session.install("-e", ".") + session.install("sphinx<3.0.0", "alabaster", "recommonmark", "sphinx-docfx-yaml") + + shutil.rmtree(os.path.join("docs", "_build"), ignore_errors=True) + session.run( + "sphinx-build", + "-T", # show full traceback on exception + "-N", # no colors + "-D", + ( + "extensions=sphinx.ext.autodoc," + "sphinx.ext.autosummary," + "docfx_yaml.extension," + "sphinx.ext.intersphinx," + "sphinx.ext.coverage," + "sphinx.ext.napoleon," + "sphinx.ext.todo," + "sphinx.ext.viewcode," + "recommonmark" + ), + "-b", + "html", + "-d", + os.path.join("docs", "_build", "doctrees", ""), + os.path.join("docs", ""), + os.path.join("docs", "_build", "html", ""), + ) diff --git a/samples/AUTHORING_GUIDE.md b/samples/AUTHORING_GUIDE.md new file mode 100644 index 00000000..55c97b32 --- /dev/null +++ b/samples/AUTHORING_GUIDE.md @@ -0,0 +1 @@ +See https://github.com/GoogleCloudPlatform/python-docs-samples/blob/master/AUTHORING_GUIDE.md \ No newline at end of file diff --git a/samples/CONTRIBUTING.md b/samples/CONTRIBUTING.md new file mode 100644 index 00000000..34c882b6 --- /dev/null +++ b/samples/CONTRIBUTING.md @@ -0,0 +1 @@ +See https://github.com/GoogleCloudPlatform/python-docs-samples/blob/master/CONTRIBUTING.md \ No newline at end of file diff --git a/samples/noxfile.py b/samples/noxfile.py new file mode 100644 index 00000000..5660f08b --- /dev/null +++ b/samples/noxfile.py @@ -0,0 +1,222 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function + +import os +from pathlib import Path +import sys + +import nox + + +# WARNING - WARNING - WARNING - WARNING - WARNING +# WARNING - WARNING - WARNING - WARNING - WARNING +# DO NOT EDIT THIS FILE EVER! +# WARNING - WARNING - WARNING - WARNING - WARNING +# WARNING - WARNING - WARNING - WARNING - WARNING + +# Copy `noxfile_config.py` to your directory and modify it instead. + + +# `TEST_CONFIG` dict is a configuration hook that allows users to +# modify the test configurations. The values here should be in sync +# with `noxfile_config.py`. Users will copy `noxfile_config.py` into +# their directory and modify it. + +TEST_CONFIG = { + # You can opt out from the test for specific Python versions. + "ignored_versions": ["2.7"], + # An envvar key for determining the project id to use. Change it + # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a + # build specific Cloud project. You can also use your own string + # to use your own Cloud project. + "gcloud_project_env": "GOOGLE_CLOUD_PROJECT", + # 'gcloud_project_env': 'BUILD_SPECIFIC_GCLOUD_PROJECT', + # A dictionary you want to inject into your test. Don't put any + # secrets here. These values will override predefined values. + "envs": {}, +} + + +try: + # Ensure we can import noxfile_config in the project's directory. + sys.path.append(".") + from noxfile_config import TEST_CONFIG_OVERRIDE +except ImportError as e: + print("No user noxfile_config found: detail: {}".format(e)) + TEST_CONFIG_OVERRIDE = {} + +# Update the TEST_CONFIG with the user supplied values. +TEST_CONFIG.update(TEST_CONFIG_OVERRIDE) + + +def get_pytest_env_vars(): + """Returns a dict for pytest invocation.""" + ret = {} + + # Override the GCLOUD_PROJECT and the alias. + env_key = TEST_CONFIG["gcloud_project_env"] + # This should error out if not set. + ret["GOOGLE_CLOUD_PROJECT"] = os.environ[env_key] + + # Apply user supplied envs. + ret.update(TEST_CONFIG["envs"]) + return ret + + +# DO NOT EDIT - automatically generated. +# All versions used to tested samples. +ALL_VERSIONS = ["2.7", "3.6", "3.7", "3.8"] + +# Any default versions that should be ignored. +IGNORED_VERSIONS = TEST_CONFIG["ignored_versions"] + +TESTED_VERSIONS = sorted([v for v in ALL_VERSIONS if v not in IGNORED_VERSIONS]) + +INSTALL_LIBRARY_FROM_SOURCE = bool(os.environ.get("INSTALL_LIBRARY_FROM_SOURCE", False)) +# +# Style Checks +# + + +def _determine_local_import_names(start_dir): + """Determines all import names that should be considered "local". + + This is used when running the linter to insure that import order is + properly checked. + """ + file_ext_pairs = [os.path.splitext(path) for path in os.listdir(start_dir)] + return [ + basename + for basename, extension in file_ext_pairs + if extension == ".py" + or os.path.isdir(os.path.join(start_dir, basename)) + and basename not in ("__pycache__") + ] + + +# Linting with flake8. +# +# We ignore the following rules: +# E203: whitespace before ‘:’ +# E266: too many leading ‘#’ for block comment +# E501: line too long +# I202: Additional newline in a section of imports +# +# We also need to specify the rules which are ignored by default: +# ['E226', 'W504', 'E126', 'E123', 'W503', 'E24', 'E704', 'E121'] +FLAKE8_COMMON_ARGS = [ + "--show-source", + "--builtin=gettext", + "--max-complexity=20", + "--import-order-style=google", + "--exclude=.nox,.cache,env,lib,generated_pb2,*_pb2.py,*_pb2_grpc.py", + "--ignore=E121,E123,E126,E203,E226,E24,E266,E501,E704,W503,W504,I202", + "--max-line-length=88", +] + + +@nox.session +def lint(session): + session.install("flake8", "flake8-import-order") + + local_names = _determine_local_import_names(".") + args = FLAKE8_COMMON_ARGS + [ + "--application-import-names", + ",".join(local_names), + ".", + ] + session.run("flake8", *args) + + +# +# Sample Tests +# + + +PYTEST_COMMON_ARGS = ["--junitxml=sponge_log.xml"] + + +def _session_tests(session, post_install=None): + """Runs py.test for a particular project.""" + if os.path.exists("requirements.txt"): + session.install("-r", "requirements.txt") + + if os.path.exists("requirements-test.txt"): + session.install("-r", "requirements-test.txt") + + if INSTALL_LIBRARY_FROM_SOURCE: + session.install("-e", _get_repo_root()) + + if post_install: + post_install(session) + + session.run( + "pytest", + *(PYTEST_COMMON_ARGS + session.posargs), + # Pytest will return 5 when no tests are collected. This can happen + # on travis where slow and flaky tests are excluded. + # See http://doc.pytest.org/en/latest/_modules/_pytest/main.html + success_codes=[0, 5], + env=get_pytest_env_vars() + ) + + +@nox.session(python=ALL_VERSIONS) +def py(session): + """Runs py.test for a sample using the specified version of Python.""" + if session.python in TESTED_VERSIONS: + _session_tests(session) + else: + session.skip( + "SKIPPED: {} tests are disabled for this sample.".format(session.python) + ) + + +# +# Readmegen +# + + +def _get_repo_root(): + """ Returns the root folder of the project. """ + # Get root of this repository. Assume we don't have directories nested deeper than 10 items. + p = Path(os.getcwd()) + for i in range(10): + if p is None: + break + if Path(p / ".git").exists(): + return str(p) + p = p.parent + raise Exception("Unable to detect repository root.") + + +GENERATED_READMES = sorted([x for x in Path(".").rglob("*.rst.in")]) + + +@nox.session +@nox.parametrize("path", GENERATED_READMES) +def readmegen(session, path): + """(Re-)generates the readme for a sample.""" + session.install("jinja2", "pyyaml") + dir_ = os.path.dirname(path) + + if os.path.exists(os.path.join(dir_, "requirements.txt")): + session.install("-r", os.path.join(dir_, "requirements.txt")) + + in_file = os.path.join(dir_, "README.rst.in") + session.run( + "python", _get_repo_root() + "/scripts/readme-gen/readme_gen.py", in_file + ) diff --git a/samples/to_dataframe/main_test.py b/samples/to_dataframe/main_test.py index 8335b437..126333bf 100644 --- a/samples/to_dataframe/main_test.py +++ b/samples/to_dataframe/main_test.py @@ -31,10 +31,7 @@ def clients(): ) # Make clients. - bqclient = bigquery.Client( - credentials=credentials, - project=your_project_id, - ) + bqclient = bigquery.Client(credentials=credentials, project=your_project_id,) bqstorageclient = bigquery_storage_v1beta1.BigQueryStorageClient( credentials=credentials ) @@ -130,9 +127,7 @@ def test_session_to_dataframe(capsys, clients): # We use a LIQUID strategy in this example because we only read from a # single stream. Consider BALANCED if you're consuming multiple streams # concurrently and want more consistent stream sizes. - sharding_strategy=( - bigquery_storage_v1beta1.enums.ShardingStrategy.LIQUID - ), + sharding_strategy=(bigquery_storage_v1beta1.enums.ShardingStrategy.LIQUID), ) # This example reads from only a single stream. Read from multiple streams diff --git a/samples/to_dataframe/noxfile.py b/samples/to_dataframe/noxfile.py new file mode 100644 index 00000000..5660f08b --- /dev/null +++ b/samples/to_dataframe/noxfile.py @@ -0,0 +1,222 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function + +import os +from pathlib import Path +import sys + +import nox + + +# WARNING - WARNING - WARNING - WARNING - WARNING +# WARNING - WARNING - WARNING - WARNING - WARNING +# DO NOT EDIT THIS FILE EVER! +# WARNING - WARNING - WARNING - WARNING - WARNING +# WARNING - WARNING - WARNING - WARNING - WARNING + +# Copy `noxfile_config.py` to your directory and modify it instead. + + +# `TEST_CONFIG` dict is a configuration hook that allows users to +# modify the test configurations. The values here should be in sync +# with `noxfile_config.py`. Users will copy `noxfile_config.py` into +# their directory and modify it. + +TEST_CONFIG = { + # You can opt out from the test for specific Python versions. + "ignored_versions": ["2.7"], + # An envvar key for determining the project id to use. Change it + # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a + # build specific Cloud project. You can also use your own string + # to use your own Cloud project. + "gcloud_project_env": "GOOGLE_CLOUD_PROJECT", + # 'gcloud_project_env': 'BUILD_SPECIFIC_GCLOUD_PROJECT', + # A dictionary you want to inject into your test. Don't put any + # secrets here. These values will override predefined values. + "envs": {}, +} + + +try: + # Ensure we can import noxfile_config in the project's directory. + sys.path.append(".") + from noxfile_config import TEST_CONFIG_OVERRIDE +except ImportError as e: + print("No user noxfile_config found: detail: {}".format(e)) + TEST_CONFIG_OVERRIDE = {} + +# Update the TEST_CONFIG with the user supplied values. +TEST_CONFIG.update(TEST_CONFIG_OVERRIDE) + + +def get_pytest_env_vars(): + """Returns a dict for pytest invocation.""" + ret = {} + + # Override the GCLOUD_PROJECT and the alias. + env_key = TEST_CONFIG["gcloud_project_env"] + # This should error out if not set. + ret["GOOGLE_CLOUD_PROJECT"] = os.environ[env_key] + + # Apply user supplied envs. + ret.update(TEST_CONFIG["envs"]) + return ret + + +# DO NOT EDIT - automatically generated. +# All versions used to tested samples. +ALL_VERSIONS = ["2.7", "3.6", "3.7", "3.8"] + +# Any default versions that should be ignored. +IGNORED_VERSIONS = TEST_CONFIG["ignored_versions"] + +TESTED_VERSIONS = sorted([v for v in ALL_VERSIONS if v not in IGNORED_VERSIONS]) + +INSTALL_LIBRARY_FROM_SOURCE = bool(os.environ.get("INSTALL_LIBRARY_FROM_SOURCE", False)) +# +# Style Checks +# + + +def _determine_local_import_names(start_dir): + """Determines all import names that should be considered "local". + + This is used when running the linter to insure that import order is + properly checked. + """ + file_ext_pairs = [os.path.splitext(path) for path in os.listdir(start_dir)] + return [ + basename + for basename, extension in file_ext_pairs + if extension == ".py" + or os.path.isdir(os.path.join(start_dir, basename)) + and basename not in ("__pycache__") + ] + + +# Linting with flake8. +# +# We ignore the following rules: +# E203: whitespace before ‘:’ +# E266: too many leading ‘#’ for block comment +# E501: line too long +# I202: Additional newline in a section of imports +# +# We also need to specify the rules which are ignored by default: +# ['E226', 'W504', 'E126', 'E123', 'W503', 'E24', 'E704', 'E121'] +FLAKE8_COMMON_ARGS = [ + "--show-source", + "--builtin=gettext", + "--max-complexity=20", + "--import-order-style=google", + "--exclude=.nox,.cache,env,lib,generated_pb2,*_pb2.py,*_pb2_grpc.py", + "--ignore=E121,E123,E126,E203,E226,E24,E266,E501,E704,W503,W504,I202", + "--max-line-length=88", +] + + +@nox.session +def lint(session): + session.install("flake8", "flake8-import-order") + + local_names = _determine_local_import_names(".") + args = FLAKE8_COMMON_ARGS + [ + "--application-import-names", + ",".join(local_names), + ".", + ] + session.run("flake8", *args) + + +# +# Sample Tests +# + + +PYTEST_COMMON_ARGS = ["--junitxml=sponge_log.xml"] + + +def _session_tests(session, post_install=None): + """Runs py.test for a particular project.""" + if os.path.exists("requirements.txt"): + session.install("-r", "requirements.txt") + + if os.path.exists("requirements-test.txt"): + session.install("-r", "requirements-test.txt") + + if INSTALL_LIBRARY_FROM_SOURCE: + session.install("-e", _get_repo_root()) + + if post_install: + post_install(session) + + session.run( + "pytest", + *(PYTEST_COMMON_ARGS + session.posargs), + # Pytest will return 5 when no tests are collected. This can happen + # on travis where slow and flaky tests are excluded. + # See http://doc.pytest.org/en/latest/_modules/_pytest/main.html + success_codes=[0, 5], + env=get_pytest_env_vars() + ) + + +@nox.session(python=ALL_VERSIONS) +def py(session): + """Runs py.test for a sample using the specified version of Python.""" + if session.python in TESTED_VERSIONS: + _session_tests(session) + else: + session.skip( + "SKIPPED: {} tests are disabled for this sample.".format(session.python) + ) + + +# +# Readmegen +# + + +def _get_repo_root(): + """ Returns the root folder of the project. """ + # Get root of this repository. Assume we don't have directories nested deeper than 10 items. + p = Path(os.getcwd()) + for i in range(10): + if p is None: + break + if Path(p / ".git").exists(): + return str(p) + p = p.parent + raise Exception("Unable to detect repository root.") + + +GENERATED_READMES = sorted([x for x in Path(".").rglob("*.rst.in")]) + + +@nox.session +@nox.parametrize("path", GENERATED_READMES) +def readmegen(session, path): + """(Re-)generates the readme for a sample.""" + session.install("jinja2", "pyyaml") + dir_ = os.path.dirname(path) + + if os.path.exists(os.path.join(dir_, "requirements.txt")): + session.install("-r", os.path.join(dir_, "requirements.txt")) + + in_file = os.path.join(dir_, "README.rst.in") + session.run( + "python", _get_repo_root() + "/scripts/readme-gen/readme_gen.py", in_file + ) diff --git a/synth.metadata b/synth.metadata index f3e781e4..8713d8fb 100644 --- a/synth.metadata +++ b/synth.metadata @@ -3,23 +3,30 @@ { "git": { "name": ".", - "remote": "https://github.com/googleapis/python-bigquery-storage.git", - "sha": "1ff6a08d5a7f17a2c8b7c40295c71a6ca7c12054" + "remote": "git@github.com:googleapis/python-bigquery-storage.git", + "sha": "9fc53559b388721cb36a48f1f6dadaeba91d1614" } }, { "git": { "name": "googleapis", "remote": "https://github.com/googleapis/googleapis.git", - "sha": "cf41866c6f14f10a07aa1e2a1260fc0a2727d889", - "internalRef": "317812187" + "sha": "2db5725bf898b544a0cf951e1694d3b0fce5eda3", + "internalRef": "329384854" } }, { "git": { "name": "synthtool", "remote": "https://github.com/googleapis/synthtool.git", - "sha": "303271797a360f8a439203413f13a160f2f5b3b4" + "sha": "d91dd8aac77f7a9c5506c238038a26fa4f9e361e" + } + }, + { + "git": { + "name": "synthtool", + "remote": "https://github.com/googleapis/synthtool.git", + "sha": "d91dd8aac77f7a9c5506c238038a26fa4f9e361e" } } ], diff --git a/synth.py b/synth.py index 3aa2312c..6cc28c93 100644 --- a/synth.py +++ b/synth.py @@ -16,6 +16,7 @@ import synthtool as s from synthtool import gcp +from synthtool.languages import python gapic = gcp.GAPICBazel() common = gcp.CommonTemplates() @@ -165,9 +166,18 @@ samples_test=True, system_test_dependencies=system_test_deps, unit_test_dependencies=optional_deps, + samples=True, ) s.move(templated_files) + +# ---------------------------------------------------------------------------- +# Samples templates +# ---------------------------------------------------------------------------- + +python.py_samples(skip_readmes=True) + + # install bigquery as a (non-editable) package s.replace( "noxfile.py", From 5407056cd53c6cab604b4ee568b2b7279eb81e07 Mon Sep 17 00:00:00 2001 From: Peter Lamut Date: Thu, 3 Sep 2020 13:48:15 +0200 Subject: [PATCH 40/42] Adjust test assertions to pyarrow values --- tests/system/v1/test_reader_v1.py | 16 ++++++++++++++-- tests/system/v1beta1/test_reader_v1beta1.py | 16 ++++++++++++++-- tests/unit/test_reader_v1.py | 5 ++++- tests/unit/test_reader_v1beta1.py | 5 ++++- 4 files changed, 36 insertions(+), 6 deletions(-) diff --git a/tests/system/v1/test_reader_v1.py b/tests/system/v1/test_reader_v1.py index df15b1a9..5337d500 100644 --- a/tests/system/v1/test_reader_v1.py +++ b/tests/system/v1/test_reader_v1.py @@ -293,7 +293,12 @@ def test_ingestion_time_partitioned_table( rows = list(client.read_rows(stream).rows(session)) assert len(rows) == 2 - actual_items = {(row["shape"], row["altitude"]) for row in rows} + if data_format == bigquery_storage_v1.enums.DataFormat.AVRO: + actual_items = {(row["shape"], row["altitude"]) for row in rows} + else: + assert data_format == bigquery_storage_v1.enums.DataFormat.ARROW + actual_items = {(row["shape"].as_py(), row["altitude"].as_py()) for row in rows} + expected_items = {("sphere", 3500), ("doughnut", 100)} assert actual_items == expected_items @@ -368,7 +373,14 @@ def test_decoding_data_types( stream = session.streams[0].name - rows = list(client.read_rows(stream).rows(session)) + if data_format == bigquery_storage_v1.enums.DataFormat.AVRO: + rows = list(client.read_rows(stream).rows(session)) + else: + assert data_format == bigquery_storage_v1.enums.DataFormat.ARROW + rows = list( + dict((key, value.as_py()) for key, value in row_dict.items()) + for row_dict in client.read_rows(stream).rows(session) + ) expected_result = { u"string_field": u"Price: € 9.95.", diff --git a/tests/system/v1beta1/test_reader_v1beta1.py b/tests/system/v1beta1/test_reader_v1beta1.py index 2ba1f99f..cd43d02c 100644 --- a/tests/system/v1beta1/test_reader_v1beta1.py +++ b/tests/system/v1beta1/test_reader_v1beta1.py @@ -306,7 +306,12 @@ def test_ingestion_time_partitioned_table( rows = list(client.read_rows(stream_pos).rows(session)) assert len(rows) == 2 - actual_items = {(row["shape"], row["altitude"]) for row in rows} + if data_format == bigquery_storage_v1beta1.enums.DataFormat.AVRO: + actual_items = {(row["shape"], row["altitude"]) for row in rows} + else: + assert data_format == bigquery_storage_v1beta1.enums.DataFormat.ARROW + actual_items = {(row["shape"].as_py(), row["altitude"].as_py()) for row in rows} + expected_items = {("sphere", 3500), ("doughnut", 100)} assert actual_items == expected_items @@ -382,7 +387,14 @@ def test_decoding_data_types( stream=session.streams[0] ) - rows = list(client.read_rows(stream_pos).rows(session)) + if data_format == bigquery_storage_v1beta1.enums.DataFormat.AVRO: + rows = list(client.read_rows(stream_pos).rows(session)) + else: + assert data_format == bigquery_storage_v1beta1.enums.DataFormat.ARROW + rows = list( + dict((key, value.as_py()) for key, value in row_dict.items()) + for row_dict in client.read_rows(stream_pos).rows(session) + ) expected_result = { u"string_field": u"Price: € 9.95.", diff --git a/tests/unit/test_reader_v1.py b/tests/unit/test_reader_v1.py index febc872d..1e67d2b4 100644 --- a/tests/unit/test_reader_v1.py +++ b/tests/unit/test_reader_v1.py @@ -326,7 +326,10 @@ def test_rows_w_scalars_arrow(class_under_test, mock_client): arrow_batches = _bq_to_arrow_batches(SCALAR_BLOCKS, arrow_schema) reader = class_under_test(arrow_batches, mock_client, "", 0, {}) - got = tuple(reader.rows(read_session)) + got = tuple( + dict((key, value.as_py()) for key, value in row_dict.items()) + for row_dict in reader.rows(read_session) + ) expected = tuple(itertools.chain.from_iterable(SCALAR_BLOCKS)) assert got == expected diff --git a/tests/unit/test_reader_v1beta1.py b/tests/unit/test_reader_v1beta1.py index f30241fa..0cded144 100644 --- a/tests/unit/test_reader_v1beta1.py +++ b/tests/unit/test_reader_v1beta1.py @@ -343,7 +343,10 @@ def test_rows_w_scalars_arrow(class_under_test, mock_client): reader = class_under_test( arrow_batches, mock_client, bigquery_storage_v1beta1.types.StreamPosition(), {} ) - got = tuple(reader.rows(read_session)) + got = tuple( + dict((key, value.as_py()) for key, value in row_dict.items()) + for row_dict in reader.rows(read_session) + ) expected = tuple(itertools.chain.from_iterable(SCALAR_BLOCKS)) assert got == expected From a1cc97289c71fe8c0bf3be940fd52bcc86b4a671 Mon Sep 17 00:00:00 2001 From: Peter Lamut Date: Thu, 3 Sep 2020 15:49:55 +0200 Subject: [PATCH 41/42] Remove samples session from the main noxfile --- noxfile.py | 17 ----------------- synth.metadata | 5 ++--- synth.py | 19 ++++++++++++++++++- 3 files changed, 20 insertions(+), 21 deletions(-) diff --git a/noxfile.py b/noxfile.py index e476c62c..39b741b9 100644 --- a/noxfile.py +++ b/noxfile.py @@ -135,23 +135,6 @@ def system(session): session.run("py.test", "--quiet", system_test_folder_path, *session.posargs) -@nox.session(python=["2.7", "3.7"]) -def samples(session): - requirements_path = os.path.join("samples", "requirements.txt") - requirements_exists = os.path.exists(requirements_path) - - # Sanity check: Only run tests if the environment variable is set. - if not os.environ.get("GOOGLE_APPLICATION_CREDENTIALS", ""): - session.skip("Credentials must be set via environment variable") - - session.install("mock", "pytest") - if requirements_exists: - session.install("-r", requirements_path) - session.install("-e", ".") - - session.run("py.test", "--quiet", "samples", *session.posargs) - - @nox.session(python=DEFAULT_PYTHON_VERSION) def cover(session): """Run the final coverage report. diff --git a/synth.metadata b/synth.metadata index 8713d8fb..7e39d2f3 100644 --- a/synth.metadata +++ b/synth.metadata @@ -4,15 +4,14 @@ "git": { "name": ".", "remote": "git@github.com:googleapis/python-bigquery-storage.git", - "sha": "9fc53559b388721cb36a48f1f6dadaeba91d1614" + "sha": "e77befee93eaffcd5378fe1ffca67c400fd2e6e0" } }, { "git": { "name": "googleapis", "remote": "https://github.com/googleapis/googleapis.git", - "sha": "2db5725bf898b544a0cf951e1694d3b0fce5eda3", - "internalRef": "329384854" + "sha": "02d3ff90a3fff5bd941ca022187ceeed4de1be4f" } }, { diff --git a/synth.py b/synth.py index 6cc28c93..56a68902 100644 --- a/synth.py +++ b/synth.py @@ -14,6 +14,8 @@ """This script is used to synthesize generated parts of this library.""" +import re + import synthtool as s from synthtool import gcp from synthtool.languages import python @@ -185,8 +187,23 @@ '\g<0>\n\n session.install("google-cloud-bigquery")', ) +# remove the samples session from the main noxfile +s.replace( + "noxfile.py", + r""" + @nox\.session\([^)]*\)\s+ + def\ samples\(session\): + .*? + (?= + @nox\.session\([^)]*\)\s+ + def\ cover\(session\): + ) + """, + "", + flags=re.VERBOSE | re.DOTALL +) + # TODO(busunkim): Use latest sphinx after microgenerator transition s.replace("noxfile.py", """['"]sphinx['"]""", '"sphinx<3.0.0"') - s.shell.run(["nox", "-s", "blacken"], hide_output=False) From 8c2fe1ad15e2c1dee1d7372048357f1955f50825 Mon Sep 17 00:00:00 2001 From: Peter Lamut Date: Thu, 3 Sep 2020 15:09:12 +0200 Subject: [PATCH 42/42] Move quickstart sample into its own subdirectory --- docs/index.rst | 2 +- samples/quickstart/__init__.py | 15 +++++++++++++++ samples/{ => quickstart}/noxfile.py | 0 samples/{ => quickstart}/quickstart.py | 0 samples/{tests => quickstart}/quickstart_test.py | 2 +- samples/quickstart/requirements-test.txt | 1 + samples/quickstart/requirements.txt | 2 ++ samples/requirements.txt | 1 - samples/tests/__init__.py | 0 9 files changed, 20 insertions(+), 3 deletions(-) create mode 100644 samples/quickstart/__init__.py rename samples/{ => quickstart}/noxfile.py (100%) rename samples/{ => quickstart}/quickstart.py (100%) rename samples/{tests => quickstart}/quickstart_test.py (97%) create mode 100644 samples/quickstart/requirements-test.txt create mode 100644 samples/quickstart/requirements.txt delete mode 100644 samples/requirements.txt delete mode 100644 samples/tests/__init__.py diff --git a/docs/index.rst b/docs/index.rst index 34918a04..ee0768ff 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -20,7 +20,7 @@ API Reference Example Usage ------------- -.. literalinclude:: samples/quickstart.py +.. literalinclude:: samples/quickstart/quickstart.py :language: python :dedent: 4 :start-after: [START bigquerystorage_quickstart] diff --git a/samples/quickstart/__init__.py b/samples/quickstart/__init__.py new file mode 100644 index 00000000..a2a70562 --- /dev/null +++ b/samples/quickstart/__init__.py @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/samples/noxfile.py b/samples/quickstart/noxfile.py similarity index 100% rename from samples/noxfile.py rename to samples/quickstart/noxfile.py diff --git a/samples/quickstart.py b/samples/quickstart/quickstart.py similarity index 100% rename from samples/quickstart.py rename to samples/quickstart/quickstart.py diff --git a/samples/tests/quickstart_test.py b/samples/quickstart/quickstart_test.py similarity index 97% rename from samples/tests/quickstart_test.py rename to samples/quickstart/quickstart_test.py index fde039f4..37b1b2dd 100644 --- a/samples/tests/quickstart_test.py +++ b/samples/quickstart/quickstart_test.py @@ -17,7 +17,7 @@ import pytest -from .. import quickstart +from . import quickstart def now_millis(): diff --git a/samples/quickstart/requirements-test.txt b/samples/quickstart/requirements-test.txt new file mode 100644 index 00000000..7e460c8c --- /dev/null +++ b/samples/quickstart/requirements-test.txt @@ -0,0 +1 @@ +pytest==6.0.1 diff --git a/samples/quickstart/requirements.txt b/samples/quickstart/requirements.txt new file mode 100644 index 00000000..31c61a34 --- /dev/null +++ b/samples/quickstart/requirements.txt @@ -0,0 +1,2 @@ +fastavro +google-cloud-bigquery-storage==1.0.0 diff --git a/samples/requirements.txt b/samples/requirements.txt deleted file mode 100644 index acd0800e..00000000 --- a/samples/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -fastavro \ No newline at end of file diff --git a/samples/tests/__init__.py b/samples/tests/__init__.py deleted file mode 100644 index e69de29b..00000000