diff --git a/samples/interactive-tutorials/events/noxfile.py b/samples/interactive-tutorials/events/noxfile.py new file mode 100644 index 00000000..f63fefdd --- /dev/null +++ b/samples/interactive-tutorials/events/noxfile.py @@ -0,0 +1,270 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function + +import os +from pathlib import Path +import sys +from typing import Callable, Dict, List, Optional + +import nox + + +# WARNING - WARNING - WARNING - WARNING - WARNING +# WARNING - WARNING - WARNING - WARNING - WARNING +# DO NOT EDIT THIS FILE EVER! +# WARNING - WARNING - WARNING - WARNING - WARNING +# WARNING - WARNING - WARNING - WARNING - WARNING + +BLACK_VERSION = "black==19.10b0" + +# Copy `noxfile_config.py` to your directory and modify it instead. + +# `TEST_CONFIG` dict is a configuration hook that allows users to +# modify the test configurations. The values here should be in sync +# with `noxfile_config.py`. Users will copy `noxfile_config.py` into +# their directory and modify it. + +TEST_CONFIG = { + # You can opt out from the test for specific Python versions. + "ignored_versions": [], + # Old samples are opted out of enforcing Python type hints + # All new samples should feature them + "enforce_type_hints": False, + # An envvar key for determining the project id to use. Change it + # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a + # build specific Cloud project. You can also use your own string + # to use your own Cloud project. + "gcloud_project_env": "GOOGLE_CLOUD_PROJECT", + # 'gcloud_project_env': 'BUILD_SPECIFIC_GCLOUD_PROJECT', + # If you need to use a specific version of pip, + # change pip_version_override to the string representation + # of the version number, for example, "20.2.4" + "pip_version_override": None, + # A dictionary you want to inject into your test. Don't put any + # secrets here. These values will override predefined values. + "envs": {}, +} + + +try: + # Ensure we can import noxfile_config in the project's directory. + sys.path.append(".") + from noxfile_config import TEST_CONFIG_OVERRIDE +except ImportError as e: + print("No user noxfile_config found: detail: {}".format(e)) + TEST_CONFIG_OVERRIDE = {} + +# Update the TEST_CONFIG with the user supplied values. +TEST_CONFIG.update(TEST_CONFIG_OVERRIDE) + + +def get_pytest_env_vars() -> Dict[str, str]: + """Returns a dict for pytest invocation.""" + ret = {} + + # Override the GCLOUD_PROJECT and the alias. + env_key = TEST_CONFIG["gcloud_project_env"] + # This should error out if not set. + ret["GOOGLE_CLOUD_PROJECT"] = os.environ[env_key] + + # Apply user supplied envs. + ret.update(TEST_CONFIG["envs"]) + return ret + + +# DO NOT EDIT - automatically generated. +# All versions used to test samples. +ALL_VERSIONS = ["3.6", "3.7", "3.8", "3.9", "3.10"] + +# Any default versions that should be ignored. +IGNORED_VERSIONS = TEST_CONFIG["ignored_versions"] + +TESTED_VERSIONS = sorted([v for v in ALL_VERSIONS if v not in IGNORED_VERSIONS]) + +INSTALL_LIBRARY_FROM_SOURCE = os.environ.get("INSTALL_LIBRARY_FROM_SOURCE", False) in ( + "True", + "true", +) + +# Error if a python version is missing +nox.options.error_on_missing_interpreters = True + +# +# Style Checks +# + + +def _determine_local_import_names(start_dir: str) -> List[str]: + """Determines all import names that should be considered "local". + This is used when running the linter to insure that import order is + properly checked. + """ + file_ext_pairs = [os.path.splitext(path) for path in os.listdir(start_dir)] + return [ + basename + for basename, extension in file_ext_pairs + if extension == ".py" + or os.path.isdir(os.path.join(start_dir, basename)) + and basename not in ("__pycache__") + ] + + +# Linting with flake8. +# +# We ignore the following rules: +# E203: whitespace before ‘:’ +# E266: too many leading ‘#’ for block comment +# E501: line too long +# I202: Additional newline in a section of imports +# +# We also need to specify the rules which are ignored by default: +# ['E226', 'W504', 'E126', 'E123', 'W503', 'E24', 'E704', 'E121'] +FLAKE8_COMMON_ARGS = [ + "--show-source", + "--builtin=gettext", + "--max-complexity=20", + "--import-order-style=google", + "--exclude=.nox,.cache,env,lib,generated_pb2,*_pb2.py,*_pb2_grpc.py", + "--ignore=E121,E123,E126,E203,E226,E24,E266,E501,E704,W503,W504,I202", + "--max-line-length=88", +] + + +@nox.session +def lint(session: nox.sessions.Session) -> None: + if not TEST_CONFIG["enforce_type_hints"]: + session.install("flake8", "flake8-import-order") + else: + session.install("flake8", "flake8-import-order", "flake8-annotations") + + local_names = _determine_local_import_names(".") + args = FLAKE8_COMMON_ARGS + [ + "--application-import-names", + ",".join(local_names), + ".", + ] + session.run("flake8", *args) + + +# +# Black +# + + +@nox.session +def blacken(session: nox.sessions.Session) -> None: + session.install(BLACK_VERSION) + python_files = [path for path in os.listdir(".") if path.endswith(".py")] + + session.run("black", *python_files) + + +# +# Sample Tests +# + + +PYTEST_COMMON_ARGS = ["--junitxml=sponge_log.xml"] + + +def _session_tests( + session: nox.sessions.Session, post_install: Callable = None +) -> None: + if TEST_CONFIG["pip_version_override"]: + pip_version = TEST_CONFIG["pip_version_override"] + session.install(f"pip=={pip_version}") + """Runs py.test for a particular project.""" + if os.path.exists("requirements.txt"): + if os.path.exists("constraints.txt"): + session.install("-r", "requirements.txt", "-c", "constraints.txt") + else: + session.install("-r", "requirements.txt") + + if os.path.exists("requirements-test.txt"): + if os.path.exists("constraints-test.txt"): + session.install("-r", "requirements-test.txt", "-c", "constraints-test.txt") + else: + session.install("-r", "requirements-test.txt") + + if INSTALL_LIBRARY_FROM_SOURCE: + session.install("-e", _get_repo_root()) + + if post_install: + post_install(session) + + session.run( + "pytest", + *(PYTEST_COMMON_ARGS + session.posargs), + # Pytest will return 5 when no tests are collected. This can happen + # on travis where slow and flaky tests are excluded. + # See http://doc.pytest.org/en/latest/_modules/_pytest/main.html + success_codes=[0, 5], + env=get_pytest_env_vars(), + ) + + +@nox.session(python=ALL_VERSIONS) +def py(session: nox.sessions.Session) -> None: + """Runs py.test for a sample using the specified version of Python.""" + if session.python in TESTED_VERSIONS: + _session_tests(session) + else: + session.skip( + "SKIPPED: {} tests are disabled for this sample.".format(session.python) + ) + + +# +# Readmegen +# + + +def _get_repo_root() -> Optional[str]: + """ Returns the root folder of the project. """ + # Get root of this repository. Assume we don't have directories nested deeper than 10 items. + p = Path(os.getcwd()) + for i in range(10): + if p is None: + break + if Path(p / ".git").exists(): + return str(p) + # .git is not available in repos cloned via Cloud Build + # setup.py is always in the library's root, so use that instead + # https://github.com/googleapis/synthtool/issues/792 + if Path(p / "setup.py").exists(): + return str(p) + p = p.parent + raise Exception("Unable to detect repository root.") + + +GENERATED_READMES = sorted([x for x in Path(".").rglob("*.rst.in")]) + + +@nox.session +@nox.parametrize("path", GENERATED_READMES) +def readmegen(session: nox.sessions.Session, path: str) -> None: + """(Re-)generates the readme for a sample.""" + session.install("jinja2", "pyyaml") + dir_ = os.path.dirname(path) + + if os.path.exists(os.path.join(dir_, "requirements.txt")): + session.install("-r", os.path.join(dir_, "requirements.txt")) + + in_file = os.path.join(dir_, "README.rst.in") + session.run( + "python", _get_repo_root() + "/scripts/readme-gen/readme_gen.py", in_file + ) + diff --git a/samples/interactive-tutorials/events/noxfile_config.py b/samples/interactive-tutorials/events/noxfile_config.py new file mode 100644 index 00000000..72486fb4 --- /dev/null +++ b/samples/interactive-tutorials/events/noxfile_config.py @@ -0,0 +1,35 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default TEST_CONFIG_OVERRIDE for python repos. + +# The source of truth: +# https://github.com/GoogleCloudPlatform/python-docs-samples/blob/master/noxfile_config.py + +TEST_CONFIG_OVERRIDE = { + # You can opt out from the test for specific Python versions. + "ignored_versions": ["2.7", "3.6", "3.8", "3.9"], + # An envvar key for determining the project id to use. Change it + # to 'PROJECT_NUMBER' if you want to opt in using a + # build specific Cloud project. You can also use your own string + # to use your own Cloud project. + # 'gcloud_project_env': 'GOOGLE_CLOUD_PROJECT', + "gcloud_project_env": "PROJECT_NUMBER", + # A dictionary you want to inject into your test. Don't put any + # secrets here. These values will override predefined values. + "envs": { + "DATA_LABELING_API_ENDPOINT": "us-central1-autopush-aiplatform.sandbox.googleapis.com", + "PYTEST_ADDOPTS": "-n=auto" # Run tests parallel using all available CPUs + }, +} diff --git a/samples/interactive-tutorials/events/purge_user_event.py b/samples/interactive-tutorials/events/purge_user_event.py new file mode 100644 index 00000000..841d4a4a --- /dev/null +++ b/samples/interactive-tutorials/events/purge_user_event.py @@ -0,0 +1,63 @@ +# Copyright 2021 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# [START retail_purge_user_event] +# Import user events into a catalog from inline source using Retail API +# +import os + +from google.api_core.client_options import ClientOptions +from google.cloud.retail import PurgeUserEventsRequest, UserEventServiceClient + +from setup.setup_cleanup import write_user_event + +project_id = os.getenv('GOOGLE_CLOUD_PROJECT') + +endpoint = "retail.googleapis.com" +default_catalog = "projects/{0}/locations/global/catalogs/default_catalog".format( + project_id) +visitor_id = 'test_visitor_id' + + +# get user events service client +def get_user_events_service_client(): + client_options = ClientOptions(endpoint) + return UserEventServiceClient(client_options=client_options) + + +# get purge user event request +def get_purge_user_event_request(): + purge_user_event_request = PurgeUserEventsRequest() + # TO CHECK ERROR HANDLING SET INVALID FILTER HERE: + purge_user_event_request.filter = 'visitorId="{}"'.format(visitor_id) + purge_user_event_request.parent = default_catalog + purge_user_event_request.force = True + print("---purge user events request---") + print(purge_user_event_request) + return purge_user_event_request + + +# call the Retail API to purge user event +def call_purge_user_events(): + purge_operation = get_user_events_service_client().purge_user_events( + get_purge_user_event_request()) + + print("---the purge operation was started:----") + print(purge_operation.operation.name) + + +write_user_event(visitor_id) +call_purge_user_events() +# [END retail_purge_user_event] diff --git a/samples/interactive-tutorials/events/purge_user_event_test.py b/samples/interactive-tutorials/events/purge_user_event_test.py new file mode 100644 index 00000000..1c51f7ab --- /dev/null +++ b/samples/interactive-tutorials/events/purge_user_event_test.py @@ -0,0 +1,36 @@ +# Copyright 2021 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re +import subprocess + + +def test_create_product(): + output = str(subprocess.check_output('python purge_user_event.py', + shell=True)) + + assert re.match('.*the user event is written.*', output) + assert re.match( + '.*purge user events request.*?parent: "projects/.*?/locations/global/catalogs/default_catalog.*', + output) + assert re.match( + '.*purge user events request.*?filter: \"visitorId=.*?test_visitor_id.*?\".*', + output) + assert re.match( + '.*purge user events request.*?parent: "projects/.*?/locations/global/catalogs/default_catalog.*', + output) + assert re.match('.*purge user events request.*?force: true.*', output) + assert re.match( + '.*the purge operation was started.*?projects/.*?/locations/global/catalogs/default_catalog/operations/purge-user-events.*', + output) diff --git a/samples/interactive-tutorials/events/rejoin_user_event.py b/samples/interactive-tutorials/events/rejoin_user_event.py new file mode 100644 index 00000000..134b708c --- /dev/null +++ b/samples/interactive-tutorials/events/rejoin_user_event.py @@ -0,0 +1,67 @@ +# Copyright 2021 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# [START retail_rejoin_user_event] +# Import user events into a catalog from inline source using Retail API +# +import os + +from google.api_core.client_options import ClientOptions +from google.cloud.retail import UserEventServiceClient, \ + RejoinUserEventsRequest + +from setup.setup_cleanup import write_user_event, purge_user_event + +project_id = os.getenv('GOOGLE_CLOUD_PROJECT') + +endpoint = "retail.googleapis.com" +default_catalog = "projects/{0}/locations/global/catalogs/default_catalog".format( + project_id) +visitor_id = 'test_visitor_id' + + +# get user events service client +def get_user_events_service_client(): + client_options = ClientOptions(endpoint) + return UserEventServiceClient(client_options=client_options) + + +# get rejoin user event request +def get_rejoin_user_event_request(): + # TO CHECK THE ERROR HANDLING TRY TO PASS INVALID CATALOG: + # default_catalog = "projects/{0}/locations/global/catalogs/invalid_catalog".format(project_number) + rejoin_user_event_request = RejoinUserEventsRequest() + rejoin_user_event_request.parent = default_catalog + rejoin_user_event_request.user_event_rejoin_scope = RejoinUserEventsRequest \ + .UserEventRejoinScope.UNJOINED_EVENTS + print("---rejoin user events request---") + print(rejoin_user_event_request) + return rejoin_user_event_request + + +# call the Retail API to rejoin user event +def call_rejoin_user_events(): + rejoin_operation = get_user_events_service_client().rejoin_user_events( + get_rejoin_user_event_request()) + + print("---the rejoin operation was started:----") + print(rejoin_operation.operation.name) + + +write_user_event(visitor_id) +call_rejoin_user_events() +purge_user_event(visitor_id) + +# [END retail_rejoin_user_event] diff --git a/samples/interactive-tutorials/events/rejoin_user_events_test.py b/samples/interactive-tutorials/events/rejoin_user_events_test.py new file mode 100644 index 00000000..f36cfff6 --- /dev/null +++ b/samples/interactive-tutorials/events/rejoin_user_events_test.py @@ -0,0 +1,35 @@ +# Copyright 2021 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re +import subprocess + + +def test_create_product(): + output = str(subprocess.check_output('python rejoin_user_event.py', + shell=True)) + + assert re.match('.*the user event is written.*', output) + assert re.match( + '.*rejoin user events request.*?parent: "projects/.*?/locations/global/catalogs/default_catalog.*', + output) + assert re.match( + '.*rejoin user events request.*?user_event_rejoin_scope: UNJOINED_EVENTS.*', + output) + assert re.match( + '.*the rejoin operation was started.*?projects/.*?/locations/global/catalogs/default_catalog/operations/rejoin-user-events.*', + output) + assert re.match( + '.*the purge operation was started.*?projects/.*?/locations/global/catalogs/default_catalog/operations/purge-user-events.*', + output) diff --git a/samples/interactive-tutorials/events/requirements-test.txt b/samples/interactive-tutorials/events/requirements-test.txt new file mode 100644 index 00000000..9299a7a8 --- /dev/null +++ b/samples/interactive-tutorials/events/requirements-test.txt @@ -0,0 +1 @@ +pytest==6.2.5 \ No newline at end of file diff --git a/samples/interactive-tutorials/events/requirements.txt b/samples/interactive-tutorials/events/requirements.txt new file mode 100644 index 00000000..0ba6ea71 --- /dev/null +++ b/samples/interactive-tutorials/events/requirements.txt @@ -0,0 +1,4 @@ +google==3.0.0 +google-cloud-retail==1.1.0 +google-cloud-storage==1.43.0 +google-cloud-bigquery==2.30.1 \ No newline at end of file diff --git a/samples/interactive-tutorials/events/setup/setup_cleanup.py b/samples/interactive-tutorials/events/setup/setup_cleanup.py new file mode 100644 index 00000000..b1337ba7 --- /dev/null +++ b/samples/interactive-tutorials/events/setup/setup_cleanup.py @@ -0,0 +1,208 @@ +# Copyright 2022 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import datetime +import json +import os +import re +import shlex +import subprocess + +from google.api_core.client_options import ClientOptions +from google.api_core.exceptions import NotFound +from google.protobuf.timestamp_pb2 import Timestamp + +from google.cloud import bigquery +from google.cloud import storage +from google.cloud.retail import ProductDetail, PurgeUserEventsRequest, \ + UserEvent, UserEventServiceClient, WriteUserEventRequest +from google.cloud.retail_v2 import Product + +project_id = os.getenv('GOOGLE_CLOUD_PROJECT') +endpoint = "retail.googleapis.com" +default_catalog = "projects/{0}/locations/global/catalogs/default_catalog".format( + project_id) + + +# get user events service client +def get_user_events_service_client(): + client_options = ClientOptions(endpoint) + return UserEventServiceClient(client_options=client_options) + + +# get user event +def get_user_event(visitor_id): + timestamp = Timestamp() + timestamp.seconds = int(datetime.datetime.now().timestamp()) + + product = Product() + product.id = 'test_id' + + product_detail = ProductDetail() + product_detail.product = product + + user_event = UserEvent() + user_event.event_type = "detail-page-view" + user_event.visitor_id = visitor_id + user_event.event_time = timestamp + user_event.product_details = [product_detail] + + print(user_event) + return user_event + + +# write user event +def write_user_event(visitor_id): + write_user_event_request = WriteUserEventRequest() + write_user_event_request.user_event = get_user_event(visitor_id) + write_user_event_request.parent = default_catalog + user_event = get_user_events_service_client().write_user_event( + write_user_event_request) + print("---the user event is written---") + print(user_event) + return user_event + + +# purge user event +def purge_user_event(visitor_id): + purge_user_event_request = PurgeUserEventsRequest() + purge_user_event_request.filter = 'visitorId="{}"'.format(visitor_id) + purge_user_event_request.parent = default_catalog + purge_user_event_request.force = True + purge_operation = get_user_events_service_client().purge_user_events( + purge_user_event_request) + + print("---the purge operation was started:----") + print(purge_operation.operation.name) + + +def get_project_id(): + get_project_command = "gcloud config get-value project --format json" + config = subprocess.check_output(shlex.split(get_project_command)) + project_id = re.search('\"(.*?)\"', str(config)).group(1) + return project_id + + +def create_bucket(bucket_name: str): + """Create a new bucket in Cloud Storage""" + print("Creating new bucket:" + bucket_name) + buckets_in_your_project = str(list_buckets()) + if bucket_name in buckets_in_your_project: + print("Bucket {} already exists".format(bucket_name)) + else: + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + bucket.storage_class = "STANDARD" + new_bucket = storage_client.create_bucket(bucket, location="us") + print( + "Created bucket {} in {} with storage class {}".format( + new_bucket.name, new_bucket.location, new_bucket.storage_class + ) + ) + return new_bucket + + +def delete_bucket(bucket_name: str): + """Delete a bucket from Cloud Storage""" + storage_client = storage.Client() + print("Deleting bucket:" + bucket_name) + buckets_in_your_project = str(list_buckets()) + if bucket_name in buckets_in_your_project: + blobs = storage_client.list_blobs(bucket_name) + for blob in blobs: + blob.delete() + bucket = storage_client.get_bucket(bucket_name) + bucket.delete() + print("Bucket {} is deleted".format(bucket.name)) + else: + print("Bucket {} is not found".format(bucket_name)) + + +def list_buckets(): + """Lists all buckets""" + bucket_list = [] + storage_client = storage.Client() + buckets = storage_client.list_buckets() + for bucket in buckets: + bucket_list.append(str(bucket)) + return bucket_list + + +def upload_blob(bucket_name, source_file_name): + """Uploads a file to the bucket.""" + # The path to your file to upload + # source_file_name = "local/path/to/file" + print("Uploading data form {} to the bucket {}".format(source_file_name, + bucket_name)) + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + object_name = re.search('resources/(.*?)$', source_file_name).group(1) + blob = bucket.blob(object_name) + blob.upload_from_filename(source_file_name) + print( + "File {} uploaded to {}.".format( + source_file_name, object_name + ) + ) + + +def create_bq_dataset(dataset_name): + """Create a BigQuery dataset""" + full_dataset_id = f"{project_id}.{dataset_name}" + bq = bigquery.Client() + print(f"Creating dataset {full_dataset_id}") + try: + bq.get_dataset(full_dataset_id) + print(f"dataset {full_dataset_id} already exists") + except NotFound: + # Construct a Dataset object to send to the API. + dataset = bq.Dataset(full_dataset_id) + dataset.location = "US" + bq.create_dataset(dataset) + print("dataset is created") + + +def create_bq_table(dataset, table_name, schema_file_path): + """Create a BigQuery table""" + full_table_id = f"{project_id}.{dataset}.{table_name}" + bq = bigquery.Client() + print(f"Creating BigQuery table {full_table_id}") + try: + bq.get_table(full_table_id) + print(f"table {full_table_id} already exists") + except NotFound: + # Construct a Table object to send to the API. + with open(schema_file_path, "rb") as schema: + schema_dict = json.load(schema) + table = bigquery.Table(full_table_id, schema=schema_dict) + bq.create_table(table) + print("table is created") + + +def upload_data_to_bq_table(dataset, table_name, source, schema_file_path): + """Upload data to the table from specified source file""" + full_table_id = f"{project_id}.{dataset}.{table_name}" + bq = bigquery.Client() + print(f"Uploading data from {source} to the table {full_table_id}") + with open(schema_file_path, "rb") as schema: + schema_dict = json.load(schema) + job_config = bigquery.LoadJobConfig( + source_format=bigquery.SourceFormat.NEWLINE_DELIMITED_JSON, + schema=schema_dict) + with open(source, "rb") as source_file: + job = bq.load_table_from_file(source_file, full_table_id, + job_config=job_config) + job.result() # Waits for the job to complete. + print("data was uploaded") \ No newline at end of file diff --git a/samples/interactive-tutorials/events/write_user_event.py b/samples/interactive-tutorials/events/write_user_event.py new file mode 100644 index 00000000..dc942695 --- /dev/null +++ b/samples/interactive-tutorials/events/write_user_event.py @@ -0,0 +1,86 @@ +# Copyright 2021 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# [START retail_write_user_event] +# Import user events into a catalog from inline source using Retail API +# +import datetime +import os + +from google.api_core.client_options import ClientOptions +from google.cloud.retail import UserEvent, UserEventServiceClient, \ + WriteUserEventRequest +from google.protobuf.timestamp_pb2 import Timestamp + +from setup.setup_cleanup import purge_user_event + +project_id = os.getenv('GOOGLE_CLOUD_PROJECT') + +endpoint = "retail.googleapis.com" +default_catalog = "projects/{0}/locations/global/catalogs/default_catalog"\ + .format(project_id) +visitor_id = 'test_visitor_id' + + +# get user events service client +def get_user_events_service_client(): + client_options = ClientOptions(endpoint) + return UserEventServiceClient(client_options=client_options) + + +# get user event +def get_user_event(): + timestamp = Timestamp() + timestamp.seconds = int(datetime.datetime.now().timestamp()) + + user_event = UserEvent() + user_event.event_type = "home-page-view" + user_event.visitor_id = visitor_id + user_event.event_time = timestamp + + print(user_event) + return user_event + + +# get write user event request +def get_write_event_request(user_event): + # TO CHECK THE ERROR HANDLING TRY TO PASS INVALID CATALOG: + # default_catalog = "projects/{0}/locations/global/catalogs/invalid_catalog" + # .format(project_number) + write_user_event_request = WriteUserEventRequest() + write_user_event_request.user_event = user_event + write_user_event_request.parent = default_catalog + + print("---write user event request---") + print(write_user_event_request) + + return write_user_event_request + + +# call the Retail API to write user event +def write_user_event(): + write_user_event_request = get_write_event_request(get_user_event()) + user_event = get_user_events_service_client().write_user_event( + write_user_event_request) + + print("---written user event:---") + print(user_event) + return user_event + + +write_user_event() +purge_user_event(visitor_id) + +# [END retail_write_user_event] diff --git a/samples/interactive-tutorials/events/write_user_event_test.py b/samples/interactive-tutorials/events/write_user_event_test.py new file mode 100644 index 00000000..41182c33 --- /dev/null +++ b/samples/interactive-tutorials/events/write_user_event_test.py @@ -0,0 +1,29 @@ +# Copyright 2021 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re +import subprocess + + +def test_create_product(): + output = str(subprocess.check_output('python write_user_event.py', + shell=True)) + + assert re.match( + '.*write user event request.*?user_event.*?event_type: "home-page-view".*', + output) + assert re.match('.*written user event.*?event_type: "home-page-view".*', + output) + assert re.match('.*written user event.*?visitor_id: "test_visitor_id".*', + output)