From d407a6a36216dba601c84619363e9f4c063ec26a Mon Sep 17 00:00:00 2001 From: tetiana-karasova <tetiana.karasova@gmail.com> Date: Thu, 17 Feb 2022 14:30:31 +0100 Subject: [PATCH 1/9] feat: User Events: write/rejoin/purge --- .../interactive-tutorials/events/noxfile.py | 270 ++++++++++++++++++ .../events/noxfile_config.py | 35 +++ .../events/purge_user_event.py | 63 ++++ .../events/purge_user_event_test.py | 36 +++ .../events/rejoin_user_event.py | 67 +++++ .../events/rejoin_user_events_test.py | 35 +++ .../events/requirements-test.txt | 1 + .../events/requirements.txt | 4 + .../events/setup/setup_cleanup.py | 208 ++++++++++++++ .../events/write_user_event.py | 86 ++++++ .../events/write_user_event_test.py | 29 ++ 11 files changed, 834 insertions(+) create mode 100644 samples/interactive-tutorials/events/noxfile.py create mode 100644 samples/interactive-tutorials/events/noxfile_config.py create mode 100644 samples/interactive-tutorials/events/purge_user_event.py create mode 100644 samples/interactive-tutorials/events/purge_user_event_test.py create mode 100644 samples/interactive-tutorials/events/rejoin_user_event.py create mode 100644 samples/interactive-tutorials/events/rejoin_user_events_test.py create mode 100644 samples/interactive-tutorials/events/requirements-test.txt create mode 100644 samples/interactive-tutorials/events/requirements.txt create mode 100644 samples/interactive-tutorials/events/setup/setup_cleanup.py create mode 100644 samples/interactive-tutorials/events/write_user_event.py create mode 100644 samples/interactive-tutorials/events/write_user_event_test.py diff --git a/samples/interactive-tutorials/events/noxfile.py b/samples/interactive-tutorials/events/noxfile.py new file mode 100644 index 00000000..f63fefdd --- /dev/null +++ b/samples/interactive-tutorials/events/noxfile.py @@ -0,0 +1,270 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function + +import os +from pathlib import Path +import sys +from typing import Callable, Dict, List, Optional + +import nox + + +# WARNING - WARNING - WARNING - WARNING - WARNING +# WARNING - WARNING - WARNING - WARNING - WARNING +# DO NOT EDIT THIS FILE EVER! +# WARNING - WARNING - WARNING - WARNING - WARNING +# WARNING - WARNING - WARNING - WARNING - WARNING + +BLACK_VERSION = "black==19.10b0" + +# Copy `noxfile_config.py` to your directory and modify it instead. + +# `TEST_CONFIG` dict is a configuration hook that allows users to +# modify the test configurations. The values here should be in sync +# with `noxfile_config.py`. Users will copy `noxfile_config.py` into +# their directory and modify it. + +TEST_CONFIG = { + # You can opt out from the test for specific Python versions. + "ignored_versions": [], + # Old samples are opted out of enforcing Python type hints + # All new samples should feature them + "enforce_type_hints": False, + # An envvar key for determining the project id to use. Change it + # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a + # build specific Cloud project. You can also use your own string + # to use your own Cloud project. + "gcloud_project_env": "GOOGLE_CLOUD_PROJECT", + # 'gcloud_project_env': 'BUILD_SPECIFIC_GCLOUD_PROJECT', + # If you need to use a specific version of pip, + # change pip_version_override to the string representation + # of the version number, for example, "20.2.4" + "pip_version_override": None, + # A dictionary you want to inject into your test. Don't put any + # secrets here. These values will override predefined values. + "envs": {}, +} + + +try: + # Ensure we can import noxfile_config in the project's directory. + sys.path.append(".") + from noxfile_config import TEST_CONFIG_OVERRIDE +except ImportError as e: + print("No user noxfile_config found: detail: {}".format(e)) + TEST_CONFIG_OVERRIDE = {} + +# Update the TEST_CONFIG with the user supplied values. +TEST_CONFIG.update(TEST_CONFIG_OVERRIDE) + + +def get_pytest_env_vars() -> Dict[str, str]: + """Returns a dict for pytest invocation.""" + ret = {} + + # Override the GCLOUD_PROJECT and the alias. + env_key = TEST_CONFIG["gcloud_project_env"] + # This should error out if not set. + ret["GOOGLE_CLOUD_PROJECT"] = os.environ[env_key] + + # Apply user supplied envs. + ret.update(TEST_CONFIG["envs"]) + return ret + + +# DO NOT EDIT - automatically generated. +# All versions used to test samples. +ALL_VERSIONS = ["3.6", "3.7", "3.8", "3.9", "3.10"] + +# Any default versions that should be ignored. +IGNORED_VERSIONS = TEST_CONFIG["ignored_versions"] + +TESTED_VERSIONS = sorted([v for v in ALL_VERSIONS if v not in IGNORED_VERSIONS]) + +INSTALL_LIBRARY_FROM_SOURCE = os.environ.get("INSTALL_LIBRARY_FROM_SOURCE", False) in ( + "True", + "true", +) + +# Error if a python version is missing +nox.options.error_on_missing_interpreters = True + +# +# Style Checks +# + + +def _determine_local_import_names(start_dir: str) -> List[str]: + """Determines all import names that should be considered "local". + This is used when running the linter to insure that import order is + properly checked. + """ + file_ext_pairs = [os.path.splitext(path) for path in os.listdir(start_dir)] + return [ + basename + for basename, extension in file_ext_pairs + if extension == ".py" + or os.path.isdir(os.path.join(start_dir, basename)) + and basename not in ("__pycache__") + ] + + +# Linting with flake8. +# +# We ignore the following rules: +# E203: whitespace before ‘:’ +# E266: too many leading ‘#’ for block comment +# E501: line too long +# I202: Additional newline in a section of imports +# +# We also need to specify the rules which are ignored by default: +# ['E226', 'W504', 'E126', 'E123', 'W503', 'E24', 'E704', 'E121'] +FLAKE8_COMMON_ARGS = [ + "--show-source", + "--builtin=gettext", + "--max-complexity=20", + "--import-order-style=google", + "--exclude=.nox,.cache,env,lib,generated_pb2,*_pb2.py,*_pb2_grpc.py", + "--ignore=E121,E123,E126,E203,E226,E24,E266,E501,E704,W503,W504,I202", + "--max-line-length=88", +] + + +@nox.session +def lint(session: nox.sessions.Session) -> None: + if not TEST_CONFIG["enforce_type_hints"]: + session.install("flake8", "flake8-import-order") + else: + session.install("flake8", "flake8-import-order", "flake8-annotations") + + local_names = _determine_local_import_names(".") + args = FLAKE8_COMMON_ARGS + [ + "--application-import-names", + ",".join(local_names), + ".", + ] + session.run("flake8", *args) + + +# +# Black +# + + +@nox.session +def blacken(session: nox.sessions.Session) -> None: + session.install(BLACK_VERSION) + python_files = [path for path in os.listdir(".") if path.endswith(".py")] + + session.run("black", *python_files) + + +# +# Sample Tests +# + + +PYTEST_COMMON_ARGS = ["--junitxml=sponge_log.xml"] + + +def _session_tests( + session: nox.sessions.Session, post_install: Callable = None +) -> None: + if TEST_CONFIG["pip_version_override"]: + pip_version = TEST_CONFIG["pip_version_override"] + session.install(f"pip=={pip_version}") + """Runs py.test for a particular project.""" + if os.path.exists("requirements.txt"): + if os.path.exists("constraints.txt"): + session.install("-r", "requirements.txt", "-c", "constraints.txt") + else: + session.install("-r", "requirements.txt") + + if os.path.exists("requirements-test.txt"): + if os.path.exists("constraints-test.txt"): + session.install("-r", "requirements-test.txt", "-c", "constraints-test.txt") + else: + session.install("-r", "requirements-test.txt") + + if INSTALL_LIBRARY_FROM_SOURCE: + session.install("-e", _get_repo_root()) + + if post_install: + post_install(session) + + session.run( + "pytest", + *(PYTEST_COMMON_ARGS + session.posargs), + # Pytest will return 5 when no tests are collected. This can happen + # on travis where slow and flaky tests are excluded. + # See http://doc.pytest.org/en/latest/_modules/_pytest/main.html + success_codes=[0, 5], + env=get_pytest_env_vars(), + ) + + +@nox.session(python=ALL_VERSIONS) +def py(session: nox.sessions.Session) -> None: + """Runs py.test for a sample using the specified version of Python.""" + if session.python in TESTED_VERSIONS: + _session_tests(session) + else: + session.skip( + "SKIPPED: {} tests are disabled for this sample.".format(session.python) + ) + + +# +# Readmegen +# + + +def _get_repo_root() -> Optional[str]: + """ Returns the root folder of the project. """ + # Get root of this repository. Assume we don't have directories nested deeper than 10 items. + p = Path(os.getcwd()) + for i in range(10): + if p is None: + break + if Path(p / ".git").exists(): + return str(p) + # .git is not available in repos cloned via Cloud Build + # setup.py is always in the library's root, so use that instead + # https://github.com/googleapis/synthtool/issues/792 + if Path(p / "setup.py").exists(): + return str(p) + p = p.parent + raise Exception("Unable to detect repository root.") + + +GENERATED_READMES = sorted([x for x in Path(".").rglob("*.rst.in")]) + + +@nox.session +@nox.parametrize("path", GENERATED_READMES) +def readmegen(session: nox.sessions.Session, path: str) -> None: + """(Re-)generates the readme for a sample.""" + session.install("jinja2", "pyyaml") + dir_ = os.path.dirname(path) + + if os.path.exists(os.path.join(dir_, "requirements.txt")): + session.install("-r", os.path.join(dir_, "requirements.txt")) + + in_file = os.path.join(dir_, "README.rst.in") + session.run( + "python", _get_repo_root() + "/scripts/readme-gen/readme_gen.py", in_file + ) + diff --git a/samples/interactive-tutorials/events/noxfile_config.py b/samples/interactive-tutorials/events/noxfile_config.py new file mode 100644 index 00000000..72486fb4 --- /dev/null +++ b/samples/interactive-tutorials/events/noxfile_config.py @@ -0,0 +1,35 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default TEST_CONFIG_OVERRIDE for python repos. + +# The source of truth: +# https://github.com/GoogleCloudPlatform/python-docs-samples/blob/master/noxfile_config.py + +TEST_CONFIG_OVERRIDE = { + # You can opt out from the test for specific Python versions. + "ignored_versions": ["2.7", "3.6", "3.8", "3.9"], + # An envvar key for determining the project id to use. Change it + # to 'PROJECT_NUMBER' if you want to opt in using a + # build specific Cloud project. You can also use your own string + # to use your own Cloud project. + # 'gcloud_project_env': 'GOOGLE_CLOUD_PROJECT', + "gcloud_project_env": "PROJECT_NUMBER", + # A dictionary you want to inject into your test. Don't put any + # secrets here. These values will override predefined values. + "envs": { + "DATA_LABELING_API_ENDPOINT": "us-central1-autopush-aiplatform.sandbox.googleapis.com", + "PYTEST_ADDOPTS": "-n=auto" # Run tests parallel using all available CPUs + }, +} diff --git a/samples/interactive-tutorials/events/purge_user_event.py b/samples/interactive-tutorials/events/purge_user_event.py new file mode 100644 index 00000000..841d4a4a --- /dev/null +++ b/samples/interactive-tutorials/events/purge_user_event.py @@ -0,0 +1,63 @@ +# Copyright 2021 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# [START retail_purge_user_event] +# Import user events into a catalog from inline source using Retail API +# +import os + +from google.api_core.client_options import ClientOptions +from google.cloud.retail import PurgeUserEventsRequest, UserEventServiceClient + +from setup.setup_cleanup import write_user_event + +project_id = os.getenv('GOOGLE_CLOUD_PROJECT') + +endpoint = "retail.googleapis.com" +default_catalog = "projects/{0}/locations/global/catalogs/default_catalog".format( + project_id) +visitor_id = 'test_visitor_id' + + +# get user events service client +def get_user_events_service_client(): + client_options = ClientOptions(endpoint) + return UserEventServiceClient(client_options=client_options) + + +# get purge user event request +def get_purge_user_event_request(): + purge_user_event_request = PurgeUserEventsRequest() + # TO CHECK ERROR HANDLING SET INVALID FILTER HERE: + purge_user_event_request.filter = 'visitorId="{}"'.format(visitor_id) + purge_user_event_request.parent = default_catalog + purge_user_event_request.force = True + print("---purge user events request---") + print(purge_user_event_request) + return purge_user_event_request + + +# call the Retail API to purge user event +def call_purge_user_events(): + purge_operation = get_user_events_service_client().purge_user_events( + get_purge_user_event_request()) + + print("---the purge operation was started:----") + print(purge_operation.operation.name) + + +write_user_event(visitor_id) +call_purge_user_events() +# [END retail_purge_user_event] diff --git a/samples/interactive-tutorials/events/purge_user_event_test.py b/samples/interactive-tutorials/events/purge_user_event_test.py new file mode 100644 index 00000000..1c51f7ab --- /dev/null +++ b/samples/interactive-tutorials/events/purge_user_event_test.py @@ -0,0 +1,36 @@ +# Copyright 2021 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re +import subprocess + + +def test_create_product(): + output = str(subprocess.check_output('python purge_user_event.py', + shell=True)) + + assert re.match('.*the user event is written.*', output) + assert re.match( + '.*purge user events request.*?parent: "projects/.*?/locations/global/catalogs/default_catalog.*', + output) + assert re.match( + '.*purge user events request.*?filter: \"visitorId=.*?test_visitor_id.*?\".*', + output) + assert re.match( + '.*purge user events request.*?parent: "projects/.*?/locations/global/catalogs/default_catalog.*', + output) + assert re.match('.*purge user events request.*?force: true.*', output) + assert re.match( + '.*the purge operation was started.*?projects/.*?/locations/global/catalogs/default_catalog/operations/purge-user-events.*', + output) diff --git a/samples/interactive-tutorials/events/rejoin_user_event.py b/samples/interactive-tutorials/events/rejoin_user_event.py new file mode 100644 index 00000000..134b708c --- /dev/null +++ b/samples/interactive-tutorials/events/rejoin_user_event.py @@ -0,0 +1,67 @@ +# Copyright 2021 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# [START retail_rejoin_user_event] +# Import user events into a catalog from inline source using Retail API +# +import os + +from google.api_core.client_options import ClientOptions +from google.cloud.retail import UserEventServiceClient, \ + RejoinUserEventsRequest + +from setup.setup_cleanup import write_user_event, purge_user_event + +project_id = os.getenv('GOOGLE_CLOUD_PROJECT') + +endpoint = "retail.googleapis.com" +default_catalog = "projects/{0}/locations/global/catalogs/default_catalog".format( + project_id) +visitor_id = 'test_visitor_id' + + +# get user events service client +def get_user_events_service_client(): + client_options = ClientOptions(endpoint) + return UserEventServiceClient(client_options=client_options) + + +# get rejoin user event request +def get_rejoin_user_event_request(): + # TO CHECK THE ERROR HANDLING TRY TO PASS INVALID CATALOG: + # default_catalog = "projects/{0}/locations/global/catalogs/invalid_catalog".format(project_number) + rejoin_user_event_request = RejoinUserEventsRequest() + rejoin_user_event_request.parent = default_catalog + rejoin_user_event_request.user_event_rejoin_scope = RejoinUserEventsRequest \ + .UserEventRejoinScope.UNJOINED_EVENTS + print("---rejoin user events request---") + print(rejoin_user_event_request) + return rejoin_user_event_request + + +# call the Retail API to rejoin user event +def call_rejoin_user_events(): + rejoin_operation = get_user_events_service_client().rejoin_user_events( + get_rejoin_user_event_request()) + + print("---the rejoin operation was started:----") + print(rejoin_operation.operation.name) + + +write_user_event(visitor_id) +call_rejoin_user_events() +purge_user_event(visitor_id) + +# [END retail_rejoin_user_event] diff --git a/samples/interactive-tutorials/events/rejoin_user_events_test.py b/samples/interactive-tutorials/events/rejoin_user_events_test.py new file mode 100644 index 00000000..f36cfff6 --- /dev/null +++ b/samples/interactive-tutorials/events/rejoin_user_events_test.py @@ -0,0 +1,35 @@ +# Copyright 2021 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re +import subprocess + + +def test_create_product(): + output = str(subprocess.check_output('python rejoin_user_event.py', + shell=True)) + + assert re.match('.*the user event is written.*', output) + assert re.match( + '.*rejoin user events request.*?parent: "projects/.*?/locations/global/catalogs/default_catalog.*', + output) + assert re.match( + '.*rejoin user events request.*?user_event_rejoin_scope: UNJOINED_EVENTS.*', + output) + assert re.match( + '.*the rejoin operation was started.*?projects/.*?/locations/global/catalogs/default_catalog/operations/rejoin-user-events.*', + output) + assert re.match( + '.*the purge operation was started.*?projects/.*?/locations/global/catalogs/default_catalog/operations/purge-user-events.*', + output) diff --git a/samples/interactive-tutorials/events/requirements-test.txt b/samples/interactive-tutorials/events/requirements-test.txt new file mode 100644 index 00000000..9299a7a8 --- /dev/null +++ b/samples/interactive-tutorials/events/requirements-test.txt @@ -0,0 +1 @@ +pytest==6.2.5 \ No newline at end of file diff --git a/samples/interactive-tutorials/events/requirements.txt b/samples/interactive-tutorials/events/requirements.txt new file mode 100644 index 00000000..0ba6ea71 --- /dev/null +++ b/samples/interactive-tutorials/events/requirements.txt @@ -0,0 +1,4 @@ +google==3.0.0 +google-cloud-retail==1.1.0 +google-cloud-storage==1.43.0 +google-cloud-bigquery==2.30.1 \ No newline at end of file diff --git a/samples/interactive-tutorials/events/setup/setup_cleanup.py b/samples/interactive-tutorials/events/setup/setup_cleanup.py new file mode 100644 index 00000000..b1337ba7 --- /dev/null +++ b/samples/interactive-tutorials/events/setup/setup_cleanup.py @@ -0,0 +1,208 @@ +# Copyright 2022 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import datetime +import json +import os +import re +import shlex +import subprocess + +from google.api_core.client_options import ClientOptions +from google.api_core.exceptions import NotFound +from google.protobuf.timestamp_pb2 import Timestamp + +from google.cloud import bigquery +from google.cloud import storage +from google.cloud.retail import ProductDetail, PurgeUserEventsRequest, \ + UserEvent, UserEventServiceClient, WriteUserEventRequest +from google.cloud.retail_v2 import Product + +project_id = os.getenv('GOOGLE_CLOUD_PROJECT') +endpoint = "retail.googleapis.com" +default_catalog = "projects/{0}/locations/global/catalogs/default_catalog".format( + project_id) + + +# get user events service client +def get_user_events_service_client(): + client_options = ClientOptions(endpoint) + return UserEventServiceClient(client_options=client_options) + + +# get user event +def get_user_event(visitor_id): + timestamp = Timestamp() + timestamp.seconds = int(datetime.datetime.now().timestamp()) + + product = Product() + product.id = 'test_id' + + product_detail = ProductDetail() + product_detail.product = product + + user_event = UserEvent() + user_event.event_type = "detail-page-view" + user_event.visitor_id = visitor_id + user_event.event_time = timestamp + user_event.product_details = [product_detail] + + print(user_event) + return user_event + + +# write user event +def write_user_event(visitor_id): + write_user_event_request = WriteUserEventRequest() + write_user_event_request.user_event = get_user_event(visitor_id) + write_user_event_request.parent = default_catalog + user_event = get_user_events_service_client().write_user_event( + write_user_event_request) + print("---the user event is written---") + print(user_event) + return user_event + + +# purge user event +def purge_user_event(visitor_id): + purge_user_event_request = PurgeUserEventsRequest() + purge_user_event_request.filter = 'visitorId="{}"'.format(visitor_id) + purge_user_event_request.parent = default_catalog + purge_user_event_request.force = True + purge_operation = get_user_events_service_client().purge_user_events( + purge_user_event_request) + + print("---the purge operation was started:----") + print(purge_operation.operation.name) + + +def get_project_id(): + get_project_command = "gcloud config get-value project --format json" + config = subprocess.check_output(shlex.split(get_project_command)) + project_id = re.search('\"(.*?)\"', str(config)).group(1) + return project_id + + +def create_bucket(bucket_name: str): + """Create a new bucket in Cloud Storage""" + print("Creating new bucket:" + bucket_name) + buckets_in_your_project = str(list_buckets()) + if bucket_name in buckets_in_your_project: + print("Bucket {} already exists".format(bucket_name)) + else: + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + bucket.storage_class = "STANDARD" + new_bucket = storage_client.create_bucket(bucket, location="us") + print( + "Created bucket {} in {} with storage class {}".format( + new_bucket.name, new_bucket.location, new_bucket.storage_class + ) + ) + return new_bucket + + +def delete_bucket(bucket_name: str): + """Delete a bucket from Cloud Storage""" + storage_client = storage.Client() + print("Deleting bucket:" + bucket_name) + buckets_in_your_project = str(list_buckets()) + if bucket_name in buckets_in_your_project: + blobs = storage_client.list_blobs(bucket_name) + for blob in blobs: + blob.delete() + bucket = storage_client.get_bucket(bucket_name) + bucket.delete() + print("Bucket {} is deleted".format(bucket.name)) + else: + print("Bucket {} is not found".format(bucket_name)) + + +def list_buckets(): + """Lists all buckets""" + bucket_list = [] + storage_client = storage.Client() + buckets = storage_client.list_buckets() + for bucket in buckets: + bucket_list.append(str(bucket)) + return bucket_list + + +def upload_blob(bucket_name, source_file_name): + """Uploads a file to the bucket.""" + # The path to your file to upload + # source_file_name = "local/path/to/file" + print("Uploading data form {} to the bucket {}".format(source_file_name, + bucket_name)) + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + object_name = re.search('resources/(.*?)$', source_file_name).group(1) + blob = bucket.blob(object_name) + blob.upload_from_filename(source_file_name) + print( + "File {} uploaded to {}.".format( + source_file_name, object_name + ) + ) + + +def create_bq_dataset(dataset_name): + """Create a BigQuery dataset""" + full_dataset_id = f"{project_id}.{dataset_name}" + bq = bigquery.Client() + print(f"Creating dataset {full_dataset_id}") + try: + bq.get_dataset(full_dataset_id) + print(f"dataset {full_dataset_id} already exists") + except NotFound: + # Construct a Dataset object to send to the API. + dataset = bq.Dataset(full_dataset_id) + dataset.location = "US" + bq.create_dataset(dataset) + print("dataset is created") + + +def create_bq_table(dataset, table_name, schema_file_path): + """Create a BigQuery table""" + full_table_id = f"{project_id}.{dataset}.{table_name}" + bq = bigquery.Client() + print(f"Creating BigQuery table {full_table_id}") + try: + bq.get_table(full_table_id) + print(f"table {full_table_id} already exists") + except NotFound: + # Construct a Table object to send to the API. + with open(schema_file_path, "rb") as schema: + schema_dict = json.load(schema) + table = bigquery.Table(full_table_id, schema=schema_dict) + bq.create_table(table) + print("table is created") + + +def upload_data_to_bq_table(dataset, table_name, source, schema_file_path): + """Upload data to the table from specified source file""" + full_table_id = f"{project_id}.{dataset}.{table_name}" + bq = bigquery.Client() + print(f"Uploading data from {source} to the table {full_table_id}") + with open(schema_file_path, "rb") as schema: + schema_dict = json.load(schema) + job_config = bigquery.LoadJobConfig( + source_format=bigquery.SourceFormat.NEWLINE_DELIMITED_JSON, + schema=schema_dict) + with open(source, "rb") as source_file: + job = bq.load_table_from_file(source_file, full_table_id, + job_config=job_config) + job.result() # Waits for the job to complete. + print("data was uploaded") \ No newline at end of file diff --git a/samples/interactive-tutorials/events/write_user_event.py b/samples/interactive-tutorials/events/write_user_event.py new file mode 100644 index 00000000..dc942695 --- /dev/null +++ b/samples/interactive-tutorials/events/write_user_event.py @@ -0,0 +1,86 @@ +# Copyright 2021 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# [START retail_write_user_event] +# Import user events into a catalog from inline source using Retail API +# +import datetime +import os + +from google.api_core.client_options import ClientOptions +from google.cloud.retail import UserEvent, UserEventServiceClient, \ + WriteUserEventRequest +from google.protobuf.timestamp_pb2 import Timestamp + +from setup.setup_cleanup import purge_user_event + +project_id = os.getenv('GOOGLE_CLOUD_PROJECT') + +endpoint = "retail.googleapis.com" +default_catalog = "projects/{0}/locations/global/catalogs/default_catalog"\ + .format(project_id) +visitor_id = 'test_visitor_id' + + +# get user events service client +def get_user_events_service_client(): + client_options = ClientOptions(endpoint) + return UserEventServiceClient(client_options=client_options) + + +# get user event +def get_user_event(): + timestamp = Timestamp() + timestamp.seconds = int(datetime.datetime.now().timestamp()) + + user_event = UserEvent() + user_event.event_type = "home-page-view" + user_event.visitor_id = visitor_id + user_event.event_time = timestamp + + print(user_event) + return user_event + + +# get write user event request +def get_write_event_request(user_event): + # TO CHECK THE ERROR HANDLING TRY TO PASS INVALID CATALOG: + # default_catalog = "projects/{0}/locations/global/catalogs/invalid_catalog" + # .format(project_number) + write_user_event_request = WriteUserEventRequest() + write_user_event_request.user_event = user_event + write_user_event_request.parent = default_catalog + + print("---write user event request---") + print(write_user_event_request) + + return write_user_event_request + + +# call the Retail API to write user event +def write_user_event(): + write_user_event_request = get_write_event_request(get_user_event()) + user_event = get_user_events_service_client().write_user_event( + write_user_event_request) + + print("---written user event:---") + print(user_event) + return user_event + + +write_user_event() +purge_user_event(visitor_id) + +# [END retail_write_user_event] diff --git a/samples/interactive-tutorials/events/write_user_event_test.py b/samples/interactive-tutorials/events/write_user_event_test.py new file mode 100644 index 00000000..41182c33 --- /dev/null +++ b/samples/interactive-tutorials/events/write_user_event_test.py @@ -0,0 +1,29 @@ +# Copyright 2021 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re +import subprocess + + +def test_create_product(): + output = str(subprocess.check_output('python write_user_event.py', + shell=True)) + + assert re.match( + '.*write user event request.*?user_event.*?event_type: "home-page-view".*', + output) + assert re.match('.*written user event.*?event_type: "home-page-view".*', + output) + assert re.match('.*written user event.*?visitor_id: "test_visitor_id".*', + output) From b8154e018fc78bf7e730e716e65b838e096d084a Mon Sep 17 00:00:00 2001 From: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com> Date: Thu, 17 Feb 2022 18:23:03 +0000 Subject: [PATCH 2/9] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- .../interactive-tutorials/events/noxfile.py | 73 +++++++++++-------- .../events/noxfile_config.py | 2 +- .../events/purge_user_event.py | 10 ++- .../events/purge_user_event_test.py | 23 +++--- .../events/rejoin_user_event.py | 18 +++-- .../events/rejoin_user_events_test.py | 23 +++--- .../events/write_user_event.py | 15 ++-- .../events/write_user_event_test.py | 12 ++- 8 files changed, 97 insertions(+), 79 deletions(-) diff --git a/samples/interactive-tutorials/events/noxfile.py b/samples/interactive-tutorials/events/noxfile.py index f63fefdd..20cdfc62 100644 --- a/samples/interactive-tutorials/events/noxfile.py +++ b/samples/interactive-tutorials/events/noxfile.py @@ -14,6 +14,7 @@ from __future__ import print_function +import glob import os from pathlib import Path import sys @@ -109,6 +110,7 @@ def get_pytest_env_vars() -> Dict[str, str]: def _determine_local_import_names(start_dir: str) -> List[str]: """Determines all import names that should be considered "local". + This is used when running the linter to insure that import order is properly checked. """ @@ -183,37 +185,45 @@ def blacken(session: nox.sessions.Session) -> None: def _session_tests( session: nox.sessions.Session, post_install: Callable = None ) -> None: - if TEST_CONFIG["pip_version_override"]: - pip_version = TEST_CONFIG["pip_version_override"] - session.install(f"pip=={pip_version}") - """Runs py.test for a particular project.""" - if os.path.exists("requirements.txt"): - if os.path.exists("constraints.txt"): - session.install("-r", "requirements.txt", "-c", "constraints.txt") - else: - session.install("-r", "requirements.txt") - - if os.path.exists("requirements-test.txt"): - if os.path.exists("constraints-test.txt"): - session.install("-r", "requirements-test.txt", "-c", "constraints-test.txt") - else: - session.install("-r", "requirements-test.txt") - - if INSTALL_LIBRARY_FROM_SOURCE: - session.install("-e", _get_repo_root()) - - if post_install: - post_install(session) - - session.run( - "pytest", - *(PYTEST_COMMON_ARGS + session.posargs), - # Pytest will return 5 when no tests are collected. This can happen - # on travis where slow and flaky tests are excluded. - # See http://doc.pytest.org/en/latest/_modules/_pytest/main.html - success_codes=[0, 5], - env=get_pytest_env_vars(), - ) + # check for presence of tests + test_list = glob.glob("*_test.py") + glob.glob("test_*.py") + test_list.extend(glob.glob("tests")) + if len(test_list) == 0: + print("No tests found, skipping directory.") + else: + if TEST_CONFIG["pip_version_override"]: + pip_version = TEST_CONFIG["pip_version_override"] + session.install(f"pip=={pip_version}") + """Runs py.test for a particular project.""" + if os.path.exists("requirements.txt"): + if os.path.exists("constraints.txt"): + session.install("-r", "requirements.txt", "-c", "constraints.txt") + else: + session.install("-r", "requirements.txt") + + if os.path.exists("requirements-test.txt"): + if os.path.exists("constraints-test.txt"): + session.install( + "-r", "requirements-test.txt", "-c", "constraints-test.txt" + ) + else: + session.install("-r", "requirements-test.txt") + + if INSTALL_LIBRARY_FROM_SOURCE: + session.install("-e", _get_repo_root()) + + if post_install: + post_install(session) + + session.run( + "pytest", + *(PYTEST_COMMON_ARGS + session.posargs), + # Pytest will return 5 when no tests are collected. This can happen + # on travis where slow and flaky tests are excluded. + # See http://doc.pytest.org/en/latest/_modules/_pytest/main.html + success_codes=[0, 5], + env=get_pytest_env_vars(), + ) @nox.session(python=ALL_VERSIONS) @@ -267,4 +277,3 @@ def readmegen(session: nox.sessions.Session, path: str) -> None: session.run( "python", _get_repo_root() + "/scripts/readme-gen/readme_gen.py", in_file ) - diff --git a/samples/interactive-tutorials/events/noxfile_config.py b/samples/interactive-tutorials/events/noxfile_config.py index 72486fb4..247d6b6c 100644 --- a/samples/interactive-tutorials/events/noxfile_config.py +++ b/samples/interactive-tutorials/events/noxfile_config.py @@ -30,6 +30,6 @@ # secrets here. These values will override predefined values. "envs": { "DATA_LABELING_API_ENDPOINT": "us-central1-autopush-aiplatform.sandbox.googleapis.com", - "PYTEST_ADDOPTS": "-n=auto" # Run tests parallel using all available CPUs + "PYTEST_ADDOPTS": "-n=auto", # Run tests parallel using all available CPUs }, } diff --git a/samples/interactive-tutorials/events/purge_user_event.py b/samples/interactive-tutorials/events/purge_user_event.py index 841d4a4a..a633b1e6 100644 --- a/samples/interactive-tutorials/events/purge_user_event.py +++ b/samples/interactive-tutorials/events/purge_user_event.py @@ -23,12 +23,13 @@ from setup.setup_cleanup import write_user_event -project_id = os.getenv('GOOGLE_CLOUD_PROJECT') +project_id = os.getenv("GOOGLE_CLOUD_PROJECT") endpoint = "retail.googleapis.com" default_catalog = "projects/{0}/locations/global/catalogs/default_catalog".format( - project_id) -visitor_id = 'test_visitor_id' + project_id +) +visitor_id = "test_visitor_id" # get user events service client @@ -52,7 +53,8 @@ def get_purge_user_event_request(): # call the Retail API to purge user event def call_purge_user_events(): purge_operation = get_user_events_service_client().purge_user_events( - get_purge_user_event_request()) + get_purge_user_event_request() + ) print("---the purge operation was started:----") print(purge_operation.operation.name) diff --git a/samples/interactive-tutorials/events/purge_user_event_test.py b/samples/interactive-tutorials/events/purge_user_event_test.py index 1c51f7ab..4eaa26e5 100644 --- a/samples/interactive-tutorials/events/purge_user_event_test.py +++ b/samples/interactive-tutorials/events/purge_user_event_test.py @@ -17,20 +17,23 @@ def test_create_product(): - output = str(subprocess.check_output('python purge_user_event.py', - shell=True)) + output = str(subprocess.check_output("python purge_user_event.py", shell=True)) - assert re.match('.*the user event is written.*', output) + assert re.match(".*the user event is written.*", output) assert re.match( '.*purge user events request.*?parent: "projects/.*?/locations/global/catalogs/default_catalog.*', - output) + output, + ) assert re.match( - '.*purge user events request.*?filter: \"visitorId=.*?test_visitor_id.*?\".*', - output) + '.*purge user events request.*?filter: "visitorId=.*?test_visitor_id.*?".*', + output, + ) assert re.match( '.*purge user events request.*?parent: "projects/.*?/locations/global/catalogs/default_catalog.*', - output) - assert re.match('.*purge user events request.*?force: true.*', output) + output, + ) + assert re.match(".*purge user events request.*?force: true.*", output) assert re.match( - '.*the purge operation was started.*?projects/.*?/locations/global/catalogs/default_catalog/operations/purge-user-events.*', - output) + ".*the purge operation was started.*?projects/.*?/locations/global/catalogs/default_catalog/operations/purge-user-events.*", + output, + ) diff --git a/samples/interactive-tutorials/events/rejoin_user_event.py b/samples/interactive-tutorials/events/rejoin_user_event.py index 134b708c..d4966f65 100644 --- a/samples/interactive-tutorials/events/rejoin_user_event.py +++ b/samples/interactive-tutorials/events/rejoin_user_event.py @@ -19,17 +19,17 @@ import os from google.api_core.client_options import ClientOptions -from google.cloud.retail import UserEventServiceClient, \ - RejoinUserEventsRequest +from google.cloud.retail import UserEventServiceClient, RejoinUserEventsRequest from setup.setup_cleanup import write_user_event, purge_user_event -project_id = os.getenv('GOOGLE_CLOUD_PROJECT') +project_id = os.getenv("GOOGLE_CLOUD_PROJECT") endpoint = "retail.googleapis.com" default_catalog = "projects/{0}/locations/global/catalogs/default_catalog".format( - project_id) -visitor_id = 'test_visitor_id' + project_id +) +visitor_id = "test_visitor_id" # get user events service client @@ -44,8 +44,9 @@ def get_rejoin_user_event_request(): # default_catalog = "projects/{0}/locations/global/catalogs/invalid_catalog".format(project_number) rejoin_user_event_request = RejoinUserEventsRequest() rejoin_user_event_request.parent = default_catalog - rejoin_user_event_request.user_event_rejoin_scope = RejoinUserEventsRequest \ - .UserEventRejoinScope.UNJOINED_EVENTS + rejoin_user_event_request.user_event_rejoin_scope = ( + RejoinUserEventsRequest.UserEventRejoinScope.UNJOINED_EVENTS + ) print("---rejoin user events request---") print(rejoin_user_event_request) return rejoin_user_event_request @@ -54,7 +55,8 @@ def get_rejoin_user_event_request(): # call the Retail API to rejoin user event def call_rejoin_user_events(): rejoin_operation = get_user_events_service_client().rejoin_user_events( - get_rejoin_user_event_request()) + get_rejoin_user_event_request() + ) print("---the rejoin operation was started:----") print(rejoin_operation.operation.name) diff --git a/samples/interactive-tutorials/events/rejoin_user_events_test.py b/samples/interactive-tutorials/events/rejoin_user_events_test.py index f36cfff6..566ba2d8 100644 --- a/samples/interactive-tutorials/events/rejoin_user_events_test.py +++ b/samples/interactive-tutorials/events/rejoin_user_events_test.py @@ -17,19 +17,22 @@ def test_create_product(): - output = str(subprocess.check_output('python rejoin_user_event.py', - shell=True)) + output = str(subprocess.check_output("python rejoin_user_event.py", shell=True)) - assert re.match('.*the user event is written.*', output) + assert re.match(".*the user event is written.*", output) assert re.match( '.*rejoin user events request.*?parent: "projects/.*?/locations/global/catalogs/default_catalog.*', - output) + output, + ) assert re.match( - '.*rejoin user events request.*?user_event_rejoin_scope: UNJOINED_EVENTS.*', - output) + ".*rejoin user events request.*?user_event_rejoin_scope: UNJOINED_EVENTS.*", + output, + ) assert re.match( - '.*the rejoin operation was started.*?projects/.*?/locations/global/catalogs/default_catalog/operations/rejoin-user-events.*', - output) + ".*the rejoin operation was started.*?projects/.*?/locations/global/catalogs/default_catalog/operations/rejoin-user-events.*", + output, + ) assert re.match( - '.*the purge operation was started.*?projects/.*?/locations/global/catalogs/default_catalog/operations/purge-user-events.*', - output) + ".*the purge operation was started.*?projects/.*?/locations/global/catalogs/default_catalog/operations/purge-user-events.*", + output, + ) diff --git a/samples/interactive-tutorials/events/write_user_event.py b/samples/interactive-tutorials/events/write_user_event.py index dc942695..dd84198a 100644 --- a/samples/interactive-tutorials/events/write_user_event.py +++ b/samples/interactive-tutorials/events/write_user_event.py @@ -20,18 +20,18 @@ import os from google.api_core.client_options import ClientOptions -from google.cloud.retail import UserEvent, UserEventServiceClient, \ - WriteUserEventRequest +from google.cloud.retail import UserEvent, UserEventServiceClient, WriteUserEventRequest from google.protobuf.timestamp_pb2 import Timestamp from setup.setup_cleanup import purge_user_event -project_id = os.getenv('GOOGLE_CLOUD_PROJECT') +project_id = os.getenv("GOOGLE_CLOUD_PROJECT") endpoint = "retail.googleapis.com" -default_catalog = "projects/{0}/locations/global/catalogs/default_catalog"\ - .format(project_id) -visitor_id = 'test_visitor_id' +default_catalog = "projects/{0}/locations/global/catalogs/default_catalog".format( + project_id +) +visitor_id = "test_visitor_id" # get user events service client @@ -73,7 +73,8 @@ def get_write_event_request(user_event): def write_user_event(): write_user_event_request = get_write_event_request(get_user_event()) user_event = get_user_events_service_client().write_user_event( - write_user_event_request) + write_user_event_request + ) print("---written user event:---") print(user_event) diff --git a/samples/interactive-tutorials/events/write_user_event_test.py b/samples/interactive-tutorials/events/write_user_event_test.py index 41182c33..3932cc7a 100644 --- a/samples/interactive-tutorials/events/write_user_event_test.py +++ b/samples/interactive-tutorials/events/write_user_event_test.py @@ -17,13 +17,11 @@ def test_create_product(): - output = str(subprocess.check_output('python write_user_event.py', - shell=True)) + output = str(subprocess.check_output("python write_user_event.py", shell=True)) assert re.match( '.*write user event request.*?user_event.*?event_type: "home-page-view".*', - output) - assert re.match('.*written user event.*?event_type: "home-page-view".*', - output) - assert re.match('.*written user event.*?visitor_id: "test_visitor_id".*', - output) + output, + ) + assert re.match('.*written user event.*?event_type: "home-page-view".*', output) + assert re.match('.*written user event.*?visitor_id: "test_visitor_id".*', output) From 2123eb722eae56b575db68ce24dc37ea8c633971 Mon Sep 17 00:00:00 2001 From: tetiana-karasova <tetiana.karasova@gmail.com> Date: Mon, 21 Feb 2022 11:43:26 +0100 Subject: [PATCH 3/9] lint fix --- samples/interactive-tutorials/events/noxfile_config.py | 3 +-- samples/interactive-tutorials/events/rejoin_user_event.py | 4 ++-- samples/interactive-tutorials/events/setup/setup_cleanup.py | 4 ++-- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/samples/interactive-tutorials/events/noxfile_config.py b/samples/interactive-tutorials/events/noxfile_config.py index 247d6b6c..0104a548 100644 --- a/samples/interactive-tutorials/events/noxfile_config.py +++ b/samples/interactive-tutorials/events/noxfile_config.py @@ -24,8 +24,7 @@ # to 'PROJECT_NUMBER' if you want to opt in using a # build specific Cloud project. You can also use your own string # to use your own Cloud project. - # 'gcloud_project_env': 'GOOGLE_CLOUD_PROJECT', - "gcloud_project_env": "PROJECT_NUMBER", + 'gcloud_project_env': 'GOOGLE_CLOUD_PROJECT', # A dictionary you want to inject into your test. Don't put any # secrets here. These values will override predefined values. "envs": { diff --git a/samples/interactive-tutorials/events/rejoin_user_event.py b/samples/interactive-tutorials/events/rejoin_user_event.py index d4966f65..e5ef5e59 100644 --- a/samples/interactive-tutorials/events/rejoin_user_event.py +++ b/samples/interactive-tutorials/events/rejoin_user_event.py @@ -19,9 +19,9 @@ import os from google.api_core.client_options import ClientOptions -from google.cloud.retail import UserEventServiceClient, RejoinUserEventsRequest +from google.cloud.retail import RejoinUserEventsRequest, UserEventServiceClient -from setup.setup_cleanup import write_user_event, purge_user_event +from setup.setup_cleanup import purge_user_event, write_user_event project_id = os.getenv("GOOGLE_CLOUD_PROJECT") diff --git a/samples/interactive-tutorials/events/setup/setup_cleanup.py b/samples/interactive-tutorials/events/setup/setup_cleanup.py index b1337ba7..e382732f 100644 --- a/samples/interactive-tutorials/events/setup/setup_cleanup.py +++ b/samples/interactive-tutorials/events/setup/setup_cleanup.py @@ -22,13 +22,13 @@ from google.api_core.client_options import ClientOptions from google.api_core.exceptions import NotFound -from google.protobuf.timestamp_pb2 import Timestamp from google.cloud import bigquery from google.cloud import storage from google.cloud.retail import ProductDetail, PurgeUserEventsRequest, \ UserEvent, UserEventServiceClient, WriteUserEventRequest from google.cloud.retail_v2 import Product +from google.protobuf.timestamp_pb2 import Timestamp project_id = os.getenv('GOOGLE_CLOUD_PROJECT') endpoint = "retail.googleapis.com" @@ -205,4 +205,4 @@ def upload_data_to_bq_table(dataset, table_name, source, schema_file_path): job = bq.load_table_from_file(source_file, full_table_id, job_config=job_config) job.result() # Waits for the job to complete. - print("data was uploaded") \ No newline at end of file + print("data was uploaded") From f339d3581cf583423dcca3996fc6a349e9c3332e Mon Sep 17 00:00:00 2001 From: tetiana-karasova <tetiana.karasova@gmail.com> Date: Mon, 21 Feb 2022 15:45:39 +0100 Subject: [PATCH 4/9] change the copyright year --- samples/interactive-tutorials/events/noxfile.py | 2 +- samples/interactive-tutorials/events/noxfile_config.py | 2 +- samples/interactive-tutorials/events/purge_user_event.py | 2 +- samples/interactive-tutorials/events/purge_user_event_test.py | 2 +- samples/interactive-tutorials/events/rejoin_user_event.py | 2 +- samples/interactive-tutorials/events/rejoin_user_events_test.py | 2 +- samples/interactive-tutorials/events/write_user_event.py | 2 +- samples/interactive-tutorials/events/write_user_event_test.py | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/samples/interactive-tutorials/events/noxfile.py b/samples/interactive-tutorials/events/noxfile.py index 20cdfc62..a2f115aa 100644 --- a/samples/interactive-tutorials/events/noxfile.py +++ b/samples/interactive-tutorials/events/noxfile.py @@ -1,4 +1,4 @@ -# Copyright 2019 Google LLC +# Copyright 2022 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/samples/interactive-tutorials/events/noxfile_config.py b/samples/interactive-tutorials/events/noxfile_config.py index 0104a548..b8a20281 100644 --- a/samples/interactive-tutorials/events/noxfile_config.py +++ b/samples/interactive-tutorials/events/noxfile_config.py @@ -1,4 +1,4 @@ -# Copyright 2020 Google LLC +# Copyright 2022 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/samples/interactive-tutorials/events/purge_user_event.py b/samples/interactive-tutorials/events/purge_user_event.py index a633b1e6..32d2e3c6 100644 --- a/samples/interactive-tutorials/events/purge_user_event.py +++ b/samples/interactive-tutorials/events/purge_user_event.py @@ -1,4 +1,4 @@ -# Copyright 2021 Google Inc. All Rights Reserved. +# Copyright 2022 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/samples/interactive-tutorials/events/purge_user_event_test.py b/samples/interactive-tutorials/events/purge_user_event_test.py index 4eaa26e5..056b883f 100644 --- a/samples/interactive-tutorials/events/purge_user_event_test.py +++ b/samples/interactive-tutorials/events/purge_user_event_test.py @@ -1,4 +1,4 @@ -# Copyright 2021 Google Inc. All Rights Reserved. +# Copyright 2022 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/samples/interactive-tutorials/events/rejoin_user_event.py b/samples/interactive-tutorials/events/rejoin_user_event.py index e5ef5e59..c85e2561 100644 --- a/samples/interactive-tutorials/events/rejoin_user_event.py +++ b/samples/interactive-tutorials/events/rejoin_user_event.py @@ -1,4 +1,4 @@ -# Copyright 2021 Google Inc. All Rights Reserved. +# Copyright 2022 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/samples/interactive-tutorials/events/rejoin_user_events_test.py b/samples/interactive-tutorials/events/rejoin_user_events_test.py index 566ba2d8..a6bea803 100644 --- a/samples/interactive-tutorials/events/rejoin_user_events_test.py +++ b/samples/interactive-tutorials/events/rejoin_user_events_test.py @@ -1,4 +1,4 @@ -# Copyright 2021 Google Inc. All Rights Reserved. +# Copyright 2022 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/samples/interactive-tutorials/events/write_user_event.py b/samples/interactive-tutorials/events/write_user_event.py index dd84198a..ee557ddb 100644 --- a/samples/interactive-tutorials/events/write_user_event.py +++ b/samples/interactive-tutorials/events/write_user_event.py @@ -1,4 +1,4 @@ -# Copyright 2021 Google Inc. All Rights Reserved. +# Copyright 2022 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/samples/interactive-tutorials/events/write_user_event_test.py b/samples/interactive-tutorials/events/write_user_event_test.py index 3932cc7a..d1dbbec7 100644 --- a/samples/interactive-tutorials/events/write_user_event_test.py +++ b/samples/interactive-tutorials/events/write_user_event_test.py @@ -1,4 +1,4 @@ -# Copyright 2021 Google Inc. All Rights Reserved. +# Copyright 2022 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. From 5020a5bc37988ed1e70c53630c1a516584d8d921 Mon Sep 17 00:00:00 2001 From: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com> Date: Mon, 21 Feb 2022 16:33:53 +0000 Subject: [PATCH 5/9] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20po?= =?UTF-8?q?st-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- samples/interactive-tutorials/events/noxfile.py | 2 +- samples/interactive-tutorials/events/noxfile_config.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/samples/interactive-tutorials/events/noxfile.py b/samples/interactive-tutorials/events/noxfile.py index a2f115aa..20cdfc62 100644 --- a/samples/interactive-tutorials/events/noxfile.py +++ b/samples/interactive-tutorials/events/noxfile.py @@ -1,4 +1,4 @@ -# Copyright 2022 Google LLC +# Copyright 2019 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/samples/interactive-tutorials/events/noxfile_config.py b/samples/interactive-tutorials/events/noxfile_config.py index b8a20281..60b418de 100644 --- a/samples/interactive-tutorials/events/noxfile_config.py +++ b/samples/interactive-tutorials/events/noxfile_config.py @@ -24,7 +24,7 @@ # to 'PROJECT_NUMBER' if you want to opt in using a # build specific Cloud project. You can also use your own string # to use your own Cloud project. - 'gcloud_project_env': 'GOOGLE_CLOUD_PROJECT', + "gcloud_project_env": "GOOGLE_CLOUD_PROJECT", # A dictionary you want to inject into your test. Don't put any # secrets here. These values will override predefined values. "envs": { From 297534c03ffbee543e24e5ccb325a457ea15205e Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou <partheniou@google.com> Date: Fri, 25 Feb 2022 12:08:39 +0000 Subject: [PATCH 6/9] fix tests --- .../events/purge_user_event.py | 2 +- .../events/rejoin_user_event.py | 2 +- ...ents_test.py => rejoin_user_event_test.py} | 0 .../events/setup/setup_cleanup.py | 208 ------------------ .../events/write_user_event.py | 2 +- 5 files changed, 3 insertions(+), 211 deletions(-) rename samples/interactive-tutorials/events/{rejoin_user_events_test.py => rejoin_user_event_test.py} (100%) delete mode 100644 samples/interactive-tutorials/events/setup/setup_cleanup.py diff --git a/samples/interactive-tutorials/events/purge_user_event.py b/samples/interactive-tutorials/events/purge_user_event.py index 32d2e3c6..d91e7b32 100644 --- a/samples/interactive-tutorials/events/purge_user_event.py +++ b/samples/interactive-tutorials/events/purge_user_event.py @@ -21,7 +21,7 @@ from google.api_core.client_options import ClientOptions from google.cloud.retail import PurgeUserEventsRequest, UserEventServiceClient -from setup.setup_cleanup import write_user_event +from setup_events.setup_cleanup import write_user_event project_id = os.getenv("GOOGLE_CLOUD_PROJECT") diff --git a/samples/interactive-tutorials/events/rejoin_user_event.py b/samples/interactive-tutorials/events/rejoin_user_event.py index c85e2561..da50865e 100644 --- a/samples/interactive-tutorials/events/rejoin_user_event.py +++ b/samples/interactive-tutorials/events/rejoin_user_event.py @@ -21,7 +21,7 @@ from google.api_core.client_options import ClientOptions from google.cloud.retail import RejoinUserEventsRequest, UserEventServiceClient -from setup.setup_cleanup import purge_user_event, write_user_event +from setup_events.setup_cleanup import purge_user_event, write_user_event project_id = os.getenv("GOOGLE_CLOUD_PROJECT") diff --git a/samples/interactive-tutorials/events/rejoin_user_events_test.py b/samples/interactive-tutorials/events/rejoin_user_event_test.py similarity index 100% rename from samples/interactive-tutorials/events/rejoin_user_events_test.py rename to samples/interactive-tutorials/events/rejoin_user_event_test.py diff --git a/samples/interactive-tutorials/events/setup/setup_cleanup.py b/samples/interactive-tutorials/events/setup/setup_cleanup.py deleted file mode 100644 index e382732f..00000000 --- a/samples/interactive-tutorials/events/setup/setup_cleanup.py +++ /dev/null @@ -1,208 +0,0 @@ -# Copyright 2022 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import datetime -import json -import os -import re -import shlex -import subprocess - -from google.api_core.client_options import ClientOptions -from google.api_core.exceptions import NotFound - -from google.cloud import bigquery -from google.cloud import storage -from google.cloud.retail import ProductDetail, PurgeUserEventsRequest, \ - UserEvent, UserEventServiceClient, WriteUserEventRequest -from google.cloud.retail_v2 import Product -from google.protobuf.timestamp_pb2 import Timestamp - -project_id = os.getenv('GOOGLE_CLOUD_PROJECT') -endpoint = "retail.googleapis.com" -default_catalog = "projects/{0}/locations/global/catalogs/default_catalog".format( - project_id) - - -# get user events service client -def get_user_events_service_client(): - client_options = ClientOptions(endpoint) - return UserEventServiceClient(client_options=client_options) - - -# get user event -def get_user_event(visitor_id): - timestamp = Timestamp() - timestamp.seconds = int(datetime.datetime.now().timestamp()) - - product = Product() - product.id = 'test_id' - - product_detail = ProductDetail() - product_detail.product = product - - user_event = UserEvent() - user_event.event_type = "detail-page-view" - user_event.visitor_id = visitor_id - user_event.event_time = timestamp - user_event.product_details = [product_detail] - - print(user_event) - return user_event - - -# write user event -def write_user_event(visitor_id): - write_user_event_request = WriteUserEventRequest() - write_user_event_request.user_event = get_user_event(visitor_id) - write_user_event_request.parent = default_catalog - user_event = get_user_events_service_client().write_user_event( - write_user_event_request) - print("---the user event is written---") - print(user_event) - return user_event - - -# purge user event -def purge_user_event(visitor_id): - purge_user_event_request = PurgeUserEventsRequest() - purge_user_event_request.filter = 'visitorId="{}"'.format(visitor_id) - purge_user_event_request.parent = default_catalog - purge_user_event_request.force = True - purge_operation = get_user_events_service_client().purge_user_events( - purge_user_event_request) - - print("---the purge operation was started:----") - print(purge_operation.operation.name) - - -def get_project_id(): - get_project_command = "gcloud config get-value project --format json" - config = subprocess.check_output(shlex.split(get_project_command)) - project_id = re.search('\"(.*?)\"', str(config)).group(1) - return project_id - - -def create_bucket(bucket_name: str): - """Create a new bucket in Cloud Storage""" - print("Creating new bucket:" + bucket_name) - buckets_in_your_project = str(list_buckets()) - if bucket_name in buckets_in_your_project: - print("Bucket {} already exists".format(bucket_name)) - else: - storage_client = storage.Client() - bucket = storage_client.bucket(bucket_name) - bucket.storage_class = "STANDARD" - new_bucket = storage_client.create_bucket(bucket, location="us") - print( - "Created bucket {} in {} with storage class {}".format( - new_bucket.name, new_bucket.location, new_bucket.storage_class - ) - ) - return new_bucket - - -def delete_bucket(bucket_name: str): - """Delete a bucket from Cloud Storage""" - storage_client = storage.Client() - print("Deleting bucket:" + bucket_name) - buckets_in_your_project = str(list_buckets()) - if bucket_name in buckets_in_your_project: - blobs = storage_client.list_blobs(bucket_name) - for blob in blobs: - blob.delete() - bucket = storage_client.get_bucket(bucket_name) - bucket.delete() - print("Bucket {} is deleted".format(bucket.name)) - else: - print("Bucket {} is not found".format(bucket_name)) - - -def list_buckets(): - """Lists all buckets""" - bucket_list = [] - storage_client = storage.Client() - buckets = storage_client.list_buckets() - for bucket in buckets: - bucket_list.append(str(bucket)) - return bucket_list - - -def upload_blob(bucket_name, source_file_name): - """Uploads a file to the bucket.""" - # The path to your file to upload - # source_file_name = "local/path/to/file" - print("Uploading data form {} to the bucket {}".format(source_file_name, - bucket_name)) - storage_client = storage.Client() - bucket = storage_client.bucket(bucket_name) - object_name = re.search('resources/(.*?)$', source_file_name).group(1) - blob = bucket.blob(object_name) - blob.upload_from_filename(source_file_name) - print( - "File {} uploaded to {}.".format( - source_file_name, object_name - ) - ) - - -def create_bq_dataset(dataset_name): - """Create a BigQuery dataset""" - full_dataset_id = f"{project_id}.{dataset_name}" - bq = bigquery.Client() - print(f"Creating dataset {full_dataset_id}") - try: - bq.get_dataset(full_dataset_id) - print(f"dataset {full_dataset_id} already exists") - except NotFound: - # Construct a Dataset object to send to the API. - dataset = bq.Dataset(full_dataset_id) - dataset.location = "US" - bq.create_dataset(dataset) - print("dataset is created") - - -def create_bq_table(dataset, table_name, schema_file_path): - """Create a BigQuery table""" - full_table_id = f"{project_id}.{dataset}.{table_name}" - bq = bigquery.Client() - print(f"Creating BigQuery table {full_table_id}") - try: - bq.get_table(full_table_id) - print(f"table {full_table_id} already exists") - except NotFound: - # Construct a Table object to send to the API. - with open(schema_file_path, "rb") as schema: - schema_dict = json.load(schema) - table = bigquery.Table(full_table_id, schema=schema_dict) - bq.create_table(table) - print("table is created") - - -def upload_data_to_bq_table(dataset, table_name, source, schema_file_path): - """Upload data to the table from specified source file""" - full_table_id = f"{project_id}.{dataset}.{table_name}" - bq = bigquery.Client() - print(f"Uploading data from {source} to the table {full_table_id}") - with open(schema_file_path, "rb") as schema: - schema_dict = json.load(schema) - job_config = bigquery.LoadJobConfig( - source_format=bigquery.SourceFormat.NEWLINE_DELIMITED_JSON, - schema=schema_dict) - with open(source, "rb") as source_file: - job = bq.load_table_from_file(source_file, full_table_id, - job_config=job_config) - job.result() # Waits for the job to complete. - print("data was uploaded") diff --git a/samples/interactive-tutorials/events/write_user_event.py b/samples/interactive-tutorials/events/write_user_event.py index ee557ddb..acbef3ab 100644 --- a/samples/interactive-tutorials/events/write_user_event.py +++ b/samples/interactive-tutorials/events/write_user_event.py @@ -23,7 +23,7 @@ from google.cloud.retail import UserEvent, UserEventServiceClient, WriteUserEventRequest from google.protobuf.timestamp_pb2 import Timestamp -from setup.setup_cleanup import purge_user_event +from setup_events.setup_cleanup import purge_user_event project_id = os.getenv("GOOGLE_CLOUD_PROJECT") From 80623096204352130237dcab63c312c9d730601f Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou <partheniou@google.com> Date: Fri, 25 Feb 2022 12:11:13 +0000 Subject: [PATCH 7/9] remove endpoint --- .../interactive-tutorials/events/purge_user_event.py | 10 +--------- .../interactive-tutorials/events/rejoin_user_event.py | 10 +--------- .../interactive-tutorials/events/write_user_event.py | 10 +--------- 3 files changed, 3 insertions(+), 27 deletions(-) diff --git a/samples/interactive-tutorials/events/purge_user_event.py b/samples/interactive-tutorials/events/purge_user_event.py index d91e7b32..31fef6f5 100644 --- a/samples/interactive-tutorials/events/purge_user_event.py +++ b/samples/interactive-tutorials/events/purge_user_event.py @@ -18,26 +18,18 @@ # import os -from google.api_core.client_options import ClientOptions from google.cloud.retail import PurgeUserEventsRequest, UserEventServiceClient from setup_events.setup_cleanup import write_user_event project_id = os.getenv("GOOGLE_CLOUD_PROJECT") -endpoint = "retail.googleapis.com" default_catalog = "projects/{0}/locations/global/catalogs/default_catalog".format( project_id ) visitor_id = "test_visitor_id" -# get user events service client -def get_user_events_service_client(): - client_options = ClientOptions(endpoint) - return UserEventServiceClient(client_options=client_options) - - # get purge user event request def get_purge_user_event_request(): purge_user_event_request = PurgeUserEventsRequest() @@ -52,7 +44,7 @@ def get_purge_user_event_request(): # call the Retail API to purge user event def call_purge_user_events(): - purge_operation = get_user_events_service_client().purge_user_events( + purge_operation = UserEventServiceClient().purge_user_events( get_purge_user_event_request() ) diff --git a/samples/interactive-tutorials/events/rejoin_user_event.py b/samples/interactive-tutorials/events/rejoin_user_event.py index da50865e..586c5ec8 100644 --- a/samples/interactive-tutorials/events/rejoin_user_event.py +++ b/samples/interactive-tutorials/events/rejoin_user_event.py @@ -18,26 +18,18 @@ # import os -from google.api_core.client_options import ClientOptions from google.cloud.retail import RejoinUserEventsRequest, UserEventServiceClient from setup_events.setup_cleanup import purge_user_event, write_user_event project_id = os.getenv("GOOGLE_CLOUD_PROJECT") -endpoint = "retail.googleapis.com" default_catalog = "projects/{0}/locations/global/catalogs/default_catalog".format( project_id ) visitor_id = "test_visitor_id" -# get user events service client -def get_user_events_service_client(): - client_options = ClientOptions(endpoint) - return UserEventServiceClient(client_options=client_options) - - # get rejoin user event request def get_rejoin_user_event_request(): # TO CHECK THE ERROR HANDLING TRY TO PASS INVALID CATALOG: @@ -54,7 +46,7 @@ def get_rejoin_user_event_request(): # call the Retail API to rejoin user event def call_rejoin_user_events(): - rejoin_operation = get_user_events_service_client().rejoin_user_events( + rejoin_operation = UserEventServiceClient().rejoin_user_events( get_rejoin_user_event_request() ) diff --git a/samples/interactive-tutorials/events/write_user_event.py b/samples/interactive-tutorials/events/write_user_event.py index acbef3ab..865d6486 100644 --- a/samples/interactive-tutorials/events/write_user_event.py +++ b/samples/interactive-tutorials/events/write_user_event.py @@ -19,7 +19,6 @@ import datetime import os -from google.api_core.client_options import ClientOptions from google.cloud.retail import UserEvent, UserEventServiceClient, WriteUserEventRequest from google.protobuf.timestamp_pb2 import Timestamp @@ -27,19 +26,12 @@ project_id = os.getenv("GOOGLE_CLOUD_PROJECT") -endpoint = "retail.googleapis.com" default_catalog = "projects/{0}/locations/global/catalogs/default_catalog".format( project_id ) visitor_id = "test_visitor_id" -# get user events service client -def get_user_events_service_client(): - client_options = ClientOptions(endpoint) - return UserEventServiceClient(client_options=client_options) - - # get user event def get_user_event(): timestamp = Timestamp() @@ -72,7 +64,7 @@ def get_write_event_request(user_event): # call the Retail API to write user event def write_user_event(): write_user_event_request = get_write_event_request(get_user_event()) - user_event = get_user_events_service_client().write_user_event( + user_event = UserEventServiceClient().write_user_event( write_user_event_request ) From c69cca5b64c8bdc2b925e19aa3b24b83c9d213f7 Mon Sep 17 00:00:00 2001 From: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com> Date: Fri, 25 Feb 2022 12:15:28 +0000 Subject: [PATCH 8/9] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20po?= =?UTF-8?q?st-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- samples/interactive-tutorials/events/write_user_event.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/samples/interactive-tutorials/events/write_user_event.py b/samples/interactive-tutorials/events/write_user_event.py index 865d6486..72984802 100644 --- a/samples/interactive-tutorials/events/write_user_event.py +++ b/samples/interactive-tutorials/events/write_user_event.py @@ -64,9 +64,7 @@ def get_write_event_request(user_event): # call the Retail API to write user event def write_user_event(): write_user_event_request = get_write_event_request(get_user_event()) - user_event = UserEventServiceClient().write_user_event( - write_user_event_request - ) + user_event = UserEventServiceClient().write_user_event(write_user_event_request) print("---written user event:---") print(user_event) From eb5e820d41f0c2b2a8427138a052375a9dbf7376 Mon Sep 17 00:00:00 2001 From: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com> Date: Fri, 25 Feb 2022 12:15:51 +0000 Subject: [PATCH 9/9] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20po?= =?UTF-8?q?st-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- samples/interactive-tutorials/events/write_user_event.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/samples/interactive-tutorials/events/write_user_event.py b/samples/interactive-tutorials/events/write_user_event.py index 865d6486..72984802 100644 --- a/samples/interactive-tutorials/events/write_user_event.py +++ b/samples/interactive-tutorials/events/write_user_event.py @@ -64,9 +64,7 @@ def get_write_event_request(user_event): # call the Retail API to write user event def write_user_event(): write_user_event_request = get_write_event_request(get_user_event()) - user_event = UserEventServiceClient().write_user_event( - write_user_event_request - ) + user_event = UserEventServiceClient().write_user_event(write_user_event_request) print("---written user event:---") print(user_event)