From ca20d5f68e30f3baa0ce234200b497e88659c5b4 Mon Sep 17 00:00:00 2001 From: Austin Parker Date: Sat, 15 Oct 2022 00:22:51 -0400 Subject: [PATCH 1/9] add cache leak failure scenario --- docker-compose.yml | 3 ++ .../20220524172636_create_featureflags.exs | 6 +++ src/recommendationservice/Dockerfile | 2 +- .../recommendation_server.py | 40 ++++++++++++++++--- 4 files changed, 45 insertions(+), 6 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index eeacfde224..cfb04057bf 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -330,9 +330,11 @@ services: depends_on: - productcatalogservice - otelcol + - featureflagservice environment: - RECOMMENDATION_SERVICE_PORT - PRODUCT_CATALOG_SERVICE_ADDR + - FEATURE_FLAG_GRPC_SERVICE_ADDR - OTEL_PYTHON_LOG_CORRELATION=true - OTEL_TRACES_EXPORTER=otlp - OTEL_METRICS_EXPORTER=otlp @@ -341,6 +343,7 @@ services: - OTEL_SERVICE_NAME=recommendationservice - PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python logging: *logging + restart: on-failure # ShippingService shippingservice: diff --git a/src/featureflagservice/priv/repo/migrations/20220524172636_create_featureflags.exs b/src/featureflagservice/priv/repo/migrations/20220524172636_create_featureflags.exs index 61dee0c5ed..c70ff7b765 100644 --- a/src/featureflagservice/priv/repo/migrations/20220524172636_create_featureflags.exs +++ b/src/featureflagservice/priv/repo/migrations/20220524172636_create_featureflags.exs @@ -25,10 +25,16 @@ defmodule Featureflagservice.Repo.Migrations.CreateFeatureflags do name: "shippingFailure", description: "Fail shipping service when shipping a product to a non-USA address", enabled: false}) + + repo().insert(%Featureflagservice.FeatureFlags.FeatureFlag{ + name: "recommendationCache", + description: "Cache recommendations", + enabled: false}) end defp execute_down do repo().delete(%Featureflagservice.FeatureFlags.FeatureFlag{name: "productCatalogFailure"}) repo().delete(%Featureflagservice.FeatureFlags.FeatureFlag{name: "shippingFailure"}) + repo().delete(%Featureflagservice.FeatureFlags.FeatureFlag{name: "recommendationCache"}) end end diff --git a/src/recommendationservice/Dockerfile b/src/recommendationservice/Dockerfile index a178ae549d..3e542988a7 100644 --- a/src/recommendationservice/Dockerfile +++ b/src/recommendationservice/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM python:3.10-slim +FROM python:3.10 WORKDIR /usr/src/app/ diff --git a/src/recommendationservice/recommendation_server.py b/src/recommendationservice/recommendation_server.py index 87c87a6286..2cdad3c3d0 100644 --- a/src/recommendationservice/recommendation_server.py +++ b/src/recommendationservice/recommendation_server.py @@ -35,6 +35,9 @@ init_metrics ) +cached_ids = [] +first_run = True + class RecommendationService(demo_pb2_grpc.RecommendationServiceServicer): def ListRecommendations(self, request, context): prod_list = get_product_list(request.product_ids) @@ -60,6 +63,8 @@ def Watch(self, request, context): def get_product_list(request_product_ids): + global first_run + global cached_ids with tracer.start_as_current_span("get_product_list") as span: max_responses = 5 @@ -67,9 +72,26 @@ def get_product_list(request_product_ids): request_product_ids_str = ''.join(request_product_ids) request_product_ids = request_product_ids_str.split(',') - # Fetch list of products from product catalog stub - cat_response = product_catalog_stub.ListProducts(demo_pb2.Empty()) - product_ids = [x.id for x in cat_response.products] + # Feature flag scenario - Cache Leak + if check_feature_flag("recommendationCache"): + if random.random() < 0.219 or first_run: + first_run = False + span.set_attribute("app.cache_hit", False) + logger.info("cache miss") + cat_response = product_catalog_stub.ListProducts(demo_pb2.Empty()) + response_ids = [x.id for x in cat_response.products] + cached_ids = cached_ids + response_ids + cached_ids = cached_ids + cached_ids + logger.info("cache length is now: " + str(len(cached_ids))) + product_ids = cached_ids + else: + span.set_attribute("app.cache_hit", True) + logger.info("cache hit") + product_ids = cached_ids + else: + cat_response = product_catalog_stub.ListProducts(demo_pb2.Empty()) + product_ids = [x.id for x in cat_response.products] + span.set_attribute("app.products.count", len(product_ids)) # Create a filtered list of products excluding the products received as input @@ -94,6 +116,11 @@ def must_map_env(key: str): raise Exception(f'{key} environment variable must be set') return value +def check_feature_flag(flag_name: str): + flag = feature_flag_stub.GetFlag(demo_pb2.GetFlagRequest(name=flag_name)).flag + logger.info(flag) + return flag.enabled + if __name__ == "__main__": # Initialize Traces and Metrics tracer = trace.get_tracer_provider().get_tracer("recommendationservice") @@ -102,9 +129,12 @@ def must_map_env(key: str): port = must_map_env('RECOMMENDATION_SERVICE_PORT') catalog_addr = must_map_env('PRODUCT_CATALOG_SERVICE_ADDR') + ff_addr = must_map_env('FEATURE_FLAG_GRPC_SERVICE_ADDR') - channel = grpc.insecure_channel(catalog_addr) - product_catalog_stub = demo_pb2_grpc.ProductCatalogServiceStub(channel) + pc_channel = grpc.insecure_channel(catalog_addr) + ff_channel = grpc.insecure_channel(ff_addr) + product_catalog_stub = demo_pb2_grpc.ProductCatalogServiceStub(pc_channel) + feature_flag_stub = demo_pb2_grpc.FeatureFlagServiceStub(ff_channel) # Create gRPC server server = grpc.server(futures.ThreadPoolExecutor(max_workers=10)) From 77bfa8d19a6d51a22fd4647c25b5f32e369b023f Mon Sep 17 00:00:00 2001 From: Austin Parker Date: Sat, 15 Oct 2022 00:27:08 -0400 Subject: [PATCH 2/9] add attribute to span --- src/recommendationservice/recommendation_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/recommendationservice/recommendation_server.py b/src/recommendationservice/recommendation_server.py index 2cdad3c3d0..f8575e4510 100644 --- a/src/recommendationservice/recommendation_server.py +++ b/src/recommendationservice/recommendation_server.py @@ -82,7 +82,7 @@ def get_product_list(request_product_ids): response_ids = [x.id for x in cat_response.products] cached_ids = cached_ids + response_ids cached_ids = cached_ids + cached_ids - logger.info("cache length is now: " + str(len(cached_ids))) + span.set_attribute("app.cache_size", len(cached_ids)) product_ids = cached_ids else: span.set_attribute("app.cache_hit", True) From ef4d93d241796a7b79320476d0f91659a3e785f3 Mon Sep 17 00:00:00 2001 From: Austin Parker Date: Sat, 15 Oct 2022 00:28:23 -0400 Subject: [PATCH 3/9] update changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a9b67fd7b..8c3ca90688 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -118,3 +118,5 @@ significant modifications will be credited to OpenTelemetry Authors. [#432](https://github.com/open-telemetry/opentelemetry-demo/pull/432) * Replaced the Jaeger exporter to the OTLP exporter in the OTel Collector ([#435](https://github.com/open-telemetry/opentelemetry-demo/pull/435)) +* Added cache scenario to recommendation service +([#455](https://github.com/open-telemetry/opentelemetry-demo/pull/455)) \ No newline at end of file From 12aaa682216eaabf9baae3c9ddc1f1bd22b09e73 Mon Sep 17 00:00:00 2001 From: Austin Parker Date: Sat, 15 Oct 2022 09:12:41 -0400 Subject: [PATCH 4/9] Update src/recommendationservice/recommendation_server.py Co-authored-by: Juliano Costa --- src/recommendationservice/recommendation_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/recommendationservice/recommendation_server.py b/src/recommendationservice/recommendation_server.py index f8575e4510..c897f566a6 100644 --- a/src/recommendationservice/recommendation_server.py +++ b/src/recommendationservice/recommendation_server.py @@ -81,7 +81,7 @@ def get_product_list(request_product_ids): cat_response = product_catalog_stub.ListProducts(demo_pb2.Empty()) response_ids = [x.id for x in cat_response.products] cached_ids = cached_ids + response_ids - cached_ids = cached_ids + cached_ids + cached_ids = cached_ids + cached_ids span.set_attribute("app.cache_size", len(cached_ids)) product_ids = cached_ids else: From aa00c749e2b9ccb6ac7b427a62e80066daa120f3 Mon Sep 17 00:00:00 2001 From: austinlparker Date: Sat, 15 Oct 2022 10:00:16 -0400 Subject: [PATCH 5/9] add newline --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8c3ca90688..51f65a0c7e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -119,4 +119,4 @@ significant modifications will be credited to OpenTelemetry Authors. * Replaced the Jaeger exporter to the OTLP exporter in the OTel Collector ([#435](https://github.com/open-telemetry/opentelemetry-demo/pull/435)) * Added cache scenario to recommendation service -([#455](https://github.com/open-telemetry/opentelemetry-demo/pull/455)) \ No newline at end of file +([#455](https://github.com/open-telemetry/opentelemetry-demo/pull/455)) From 12052847bc71d6f1200a5f9717f1620892f0015a Mon Sep 17 00:00:00 2001 From: austinlparker Date: Sat, 15 Oct 2022 10:02:42 -0400 Subject: [PATCH 6/9] add docs --- docs/manual_span_attributes.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/manual_span_attributes.md b/docs/manual_span_attributes.md index 72648960b1..4f79dac724 100644 --- a/docs/manual_span_attributes.md +++ b/docs/manual_span_attributes.md @@ -107,11 +107,12 @@ This document contains the list of manual Span Attributes used throughout the de ## RecommendationService -| Name | Type | Description | -|----------------------------------|--------|-----------------------------------------| -| `app.filtered_products.count` | number | Number of filtered products returned | -| `app.products.count` | number | Number of products in catalog | -| `app.products_recommended.count` | number | Number of recommended products returned | +| Name | Type | Description | +|----------------------------------|---------|-----------------------------------------| +| `app.filtered_products.count` | number | Number of filtered products returned | +| `app.products.count` | number | Number of products in catalog | +| `app.products_recommended.count` | number | Number of recommended products returned | +| `app.cache_hit` | boolean | If cache was accessed or not | ## ShippingService From 882ad0ece35a0ca8ec647d64c2af40b26af23851 Mon Sep 17 00:00:00 2001 From: Austin Parker Date: Sat, 15 Oct 2022 22:29:05 -0400 Subject: [PATCH 7/9] tweak scenario --- src/recommendationservice/recommendation_server.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/recommendationservice/recommendation_server.py b/src/recommendationservice/recommendation_server.py index c897f566a6..607dc519e8 100644 --- a/src/recommendationservice/recommendation_server.py +++ b/src/recommendationservice/recommendation_server.py @@ -74,14 +74,14 @@ def get_product_list(request_product_ids): # Feature flag scenario - Cache Leak if check_feature_flag("recommendationCache"): - if random.random() < 0.219 or first_run: + if random.random() < 0.5 or first_run: first_run = False span.set_attribute("app.cache_hit", False) logger.info("cache miss") cat_response = product_catalog_stub.ListProducts(demo_pb2.Empty()) response_ids = [x.id for x in cat_response.products] cached_ids = cached_ids + response_ids - cached_ids = cached_ids + cached_ids + cached_ids = cached_ids + cached_ids[:len(cached_ids) // 4] span.set_attribute("app.cache_size", len(cached_ids)) product_ids = cached_ids else: From 32705a82cdcbf3102e14672cc08865e71eb5697d Mon Sep 17 00:00:00 2001 From: Austin Parker Date: Sun, 16 Oct 2022 13:49:46 -0400 Subject: [PATCH 8/9] add resource limit to force service overlimit --- docker-compose.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index cfb04057bf..4ae3e60f29 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -344,6 +344,10 @@ services: - PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python logging: *logging restart: on-failure + deploy: + resources: + limits: + memory: 512M # ShippingService shippingservice: From 0d236ce43987a87732297c7d295d15375391a9e2 Mon Sep 17 00:00:00 2001 From: Austin Parker Date: Mon, 17 Oct 2022 11:47:27 -0400 Subject: [PATCH 9/9] review --- src/recommendationservice/recommendation_server.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/recommendationservice/recommendation_server.py b/src/recommendationservice/recommendation_server.py index 607dc519e8..883c7cb823 100644 --- a/src/recommendationservice/recommendation_server.py +++ b/src/recommendationservice/recommendation_server.py @@ -74,6 +74,7 @@ def get_product_list(request_product_ids): # Feature flag scenario - Cache Leak if check_feature_flag("recommendationCache"): + span.set_attribute("app.recommendation.cache_enabled", True) if random.random() < 0.5 or first_run: first_run = False span.set_attribute("app.cache_hit", False) @@ -82,13 +83,13 @@ def get_product_list(request_product_ids): response_ids = [x.id for x in cat_response.products] cached_ids = cached_ids + response_ids cached_ids = cached_ids + cached_ids[:len(cached_ids) // 4] - span.set_attribute("app.cache_size", len(cached_ids)) product_ids = cached_ids else: span.set_attribute("app.cache_hit", True) logger.info("cache hit") product_ids = cached_ids else: + span.set_attribute("app.recommendation.cache_enabled", False) cat_response = product_catalog_stub.ListProducts(demo_pb2.Empty()) product_ids = [x.id for x in cat_response.products]